groove/Part-DB-server
1
0
Fork 0
mirror of https://github.com/Part-DB/Part-DB-server.git synced 2025-12-06 02:59:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Optimized LCSC batch search calls and extracted it into interface for potential general use in the future

Browse source
This commit is contained in:
Jan Böhmer 2025-08-31 23:41:16 +02:00
parent 4fcd55748f
commit 52444e05e4
3 changed files with 60 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

46
src/Controller/BulkInfoProviderImportController.php
View file

@ -30,6 +30,8 @@ use App\Entity\Parts\Supplier;
use App\Form\InfoProviderSystem\GlobalFieldMappingType;
use App\Services\InfoProviderSystem\PartInfoRetriever;
use App\Services\InfoProviderSystem\ExistingPartFinder;
use App\Services\InfoProviderSystem\ProviderRegistry;
use App\Services\InfoProviderSystem\Providers\LCSCProvider;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
@ -44,6 +46,7 @@ class BulkInfoProviderImportController extends AbstractController
{
public function __construct(
private readonly PartInfoRetriever $infoRetriever,
private readonly LCSCProvider $LCSCProvider,
private readonly ExistingPartFinder $existingPartFinder,
private readonly EntityManagerInterface $entityManager
) {
@ -53,7 +56,7 @@ class BulkInfoProviderImportController extends AbstractController
public function step1(Request $request, LoggerInterface $exceptionLogger): Response
{
$this->denyAccessUnlessGranted('@info_providers.create_parts');
// Increase execution time for bulk operations
set_time_limit(600); // 10 minutes for large batches
@ -72,7 +75,7 @@ class BulkInfoProviderImportController extends AbstractController
$this->addFlash('error', 'No valid parts found for bulk import');
return $this->redirectToRoute('homepage');
}
// Warn about large batches
if (count($parts) > 50) {
$this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.');
@ -110,7 +113,7 @@ class BulkInfoProviderImportController extends AbstractController
$formData = $form->getData();
$fieldMappings = $formData['field_mappings'];
$prefetchDetails = $formData['prefetch_details'] ?? false;
// Debug logging
$exceptionLogger->info('Form data received', [
'prefetch_details' => $prefetchDetails,
@ -143,13 +146,13 @@ class BulkInfoProviderImportController extends AbstractController
// Optimize: Use batch async requests for LCSC provider
$lcscKeywords = [];
$keywordToPartField = [];
// First, collect all LCSC keywords for batch processing
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
$field = $mapping['field'];
$providers = $mapping['providers'] ?? [];
if (in_array('lcsc', $providers, true)) {
$keyword = $this->getKeywordFromField($part, $field);
if ($keyword) {
@ -291,11 +294,11 @@ class BulkInfoProviderImportController extends AbstractController
'job_id' => $job->getId(),
'parts_count' => count($parts)
]);
// Delete the job since it has no useful results
$this->entityManager->remove($job);
$this->entityManager->flush();
$this->addFlash('error', 'No search results found for any of the selected parts. Please check your field mappings and provider selections.');
return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
}
@ -304,18 +307,18 @@ class BulkInfoProviderImportController extends AbstractController
$job->setSearchResults($this->serializeSearchResults($searchResults));
$job->markAsInProgress();
$this->entityManager->flush();
} catch (\Exception $e) {
$exceptionLogger->error('Critical error during bulk import search', [
'job_id' => $job->getId(),
'error' => $e->getMessage(),
'exception' => $e
]);
// Delete the job on critical failure
$this->entityManager->remove($job);
$this->entityManager->flush();
$this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage());
return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
}
@ -356,19 +359,19 @@ class BulkInfoProviderImportController extends AbstractController
// Also clean up jobs with no results (failed searches)
$updatedJobs = false;
$jobsToDelete = [];
foreach ($allJobs as $job) {
if ($job->isAllPartsCompleted() && !$job->isCompleted()) {
$job->markAsCompleted();
$updatedJobs = true;
}
// Mark jobs with no results for deletion (failed searches)
if ($job->getResultCount() === 0 && $job->isInProgress()) {
$jobsToDelete[] = $job;
}
}
// Delete failed jobs
foreach ($jobsToDelete as $job) {
$this->entityManager->remove($job);
@ -378,7 +381,7 @@ class BulkInfoProviderImportController extends AbstractController
// Flush changes if any jobs were updated
if ($updatedJobs) {
$this->entityManager->flush();
if (!empty($jobsToDelete)) {
$this->addFlash('info', 'Cleaned up ' . count($jobsToDelete) . ' failed job(s) with no results.');
}
@ -619,18 +622,7 @@ class BulkInfoProviderImportController extends AbstractController
*/
private function searchLcscBatch(array $keywords): array
{
// Get LCSC provider through reflection since PartInfoRetriever doesn't expose it
$reflection = new \ReflectionClass($this->infoRetriever);
$registryProp = $reflection->getProperty('provider_registry');
$registryProp->setAccessible(true);
$registry = $registryProp->getValue($this->infoRetriever);
$lcscProvider = $registry->getProviderByKey('lcsc');
if ($lcscProvider && method_exists($lcscProvider, 'searchByKeywordsBatch')) {
return $lcscProvider->searchByKeywordsBatch($keywords);
}
return [];
return $this->LCSCProvider->searchByKeywordsBatch($keywords);
}
#[Route('/job/{jobId}/part/{partId}/mark-completed', name: 'bulk_info_provider_mark_completed', methods: ['POST'])]
@ -710,4 +702,4 @@ class BulkInfoProviderImportController extends AbstractController
'job_completed' => $job->isCompleted()
]);
}
}
}

40
src/Services/InfoProviderSystem/Providers/BatchInfoProviderInterface.php Normal file
View file

@ -0,0 +1,40 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\Providers;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
/**
* This interface marks a provider as a info provider which can provide information directly in batch operations
*/
interface BatchInfoProviderInterface extends InfoProviderInterface
{
/**
* Search for multiple keywords in a single batch operation and return the results, ordered by the keywords.
* This allows for a more efficient search compared to running multiple single searches.
* @param string[] $keywords
* @return array<string, SearchResultDTO[]> An associative array where the key is the keyword and the value is the search results for that keyword
*/
public function searchByKeywordsBatch(array $keywords): array;
}

2
src/Services/InfoProviderSystem/Providers/LCSCProvider.php
View file

@ -33,7 +33,7 @@ use App\Settings\InfoProviderSystem\LCSCSettings;
use Symfony\Component\HttpFoundation\Cookie;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class LCSCProvider implements InfoProviderInterface
class LCSCProvider implements BatchInfoProviderInterface
{
private const ENDPOINT_URL = 'https://wmsc.lcsc.com/ftps/wm';