From 52444e05e40a5c4948032d20fe2b49eb0abfce75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Sun, 31 Aug 2025 23:41:16 +0200 Subject: [PATCH] Optimized LCSC batch search calls and extracted it into interface for potential general use in the future --- .../BulkInfoProviderImportController.php | 46 ++++++++----------- .../Providers/BatchInfoProviderInterface.php | 40 ++++++++++++++++ .../Providers/LCSCProvider.php | 2 +- 3 files changed, 60 insertions(+), 28 deletions(-) create mode 100644 src/Services/InfoProviderSystem/Providers/BatchInfoProviderInterface.php diff --git a/src/Controller/BulkInfoProviderImportController.php b/src/Controller/BulkInfoProviderImportController.php index d09e8d04..880bfe5f 100644 --- a/src/Controller/BulkInfoProviderImportController.php +++ b/src/Controller/BulkInfoProviderImportController.php @@ -30,6 +30,8 @@ use App\Entity\Parts\Supplier; use App\Form\InfoProviderSystem\GlobalFieldMappingType; use App\Services\InfoProviderSystem\PartInfoRetriever; use App\Services\InfoProviderSystem\ExistingPartFinder; +use App\Services\InfoProviderSystem\ProviderRegistry; +use App\Services\InfoProviderSystem\Providers\LCSCProvider; use Doctrine\ORM\EntityManagerInterface; use Psr\Log\LoggerInterface; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; @@ -44,6 +46,7 @@ class BulkInfoProviderImportController extends AbstractController { public function __construct( private readonly PartInfoRetriever $infoRetriever, + private readonly LCSCProvider $LCSCProvider, private readonly ExistingPartFinder $existingPartFinder, private readonly EntityManagerInterface $entityManager ) { @@ -53,7 +56,7 @@ class BulkInfoProviderImportController extends AbstractController public function step1(Request $request, LoggerInterface $exceptionLogger): Response { $this->denyAccessUnlessGranted('@info_providers.create_parts'); - + // Increase execution time for bulk operations set_time_limit(600); // 10 minutes for large batches @@ -72,7 +75,7 @@ class BulkInfoProviderImportController extends AbstractController $this->addFlash('error', 'No valid parts found for bulk import'); return $this->redirectToRoute('homepage'); } - + // Warn about large batches if (count($parts) > 50) { $this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.'); @@ -110,7 +113,7 @@ class BulkInfoProviderImportController extends AbstractController $formData = $form->getData(); $fieldMappings = $formData['field_mappings']; $prefetchDetails = $formData['prefetch_details'] ?? false; - + // Debug logging $exceptionLogger->info('Form data received', [ 'prefetch_details' => $prefetchDetails, @@ -143,13 +146,13 @@ class BulkInfoProviderImportController extends AbstractController // Optimize: Use batch async requests for LCSC provider $lcscKeywords = []; $keywordToPartField = []; - + // First, collect all LCSC keywords for batch processing foreach ($parts as $part) { foreach ($fieldMappings as $mapping) { $field = $mapping['field']; $providers = $mapping['providers'] ?? []; - + if (in_array('lcsc', $providers, true)) { $keyword = $this->getKeywordFromField($part, $field); if ($keyword) { @@ -291,11 +294,11 @@ class BulkInfoProviderImportController extends AbstractController 'job_id' => $job->getId(), 'parts_count' => count($parts) ]); - + // Delete the job since it has no useful results $this->entityManager->remove($job); $this->entityManager->flush(); - + $this->addFlash('error', 'No search results found for any of the selected parts. Please check your field mappings and provider selections.'); return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]); } @@ -304,18 +307,18 @@ class BulkInfoProviderImportController extends AbstractController $job->setSearchResults($this->serializeSearchResults($searchResults)); $job->markAsInProgress(); $this->entityManager->flush(); - + } catch (\Exception $e) { $exceptionLogger->error('Critical error during bulk import search', [ 'job_id' => $job->getId(), 'error' => $e->getMessage(), 'exception' => $e ]); - + // Delete the job on critical failure $this->entityManager->remove($job); $this->entityManager->flush(); - + $this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage()); return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]); } @@ -356,19 +359,19 @@ class BulkInfoProviderImportController extends AbstractController // Also clean up jobs with no results (failed searches) $updatedJobs = false; $jobsToDelete = []; - + foreach ($allJobs as $job) { if ($job->isAllPartsCompleted() && !$job->isCompleted()) { $job->markAsCompleted(); $updatedJobs = true; } - + // Mark jobs with no results for deletion (failed searches) if ($job->getResultCount() === 0 && $job->isInProgress()) { $jobsToDelete[] = $job; } } - + // Delete failed jobs foreach ($jobsToDelete as $job) { $this->entityManager->remove($job); @@ -378,7 +381,7 @@ class BulkInfoProviderImportController extends AbstractController // Flush changes if any jobs were updated if ($updatedJobs) { $this->entityManager->flush(); - + if (!empty($jobsToDelete)) { $this->addFlash('info', 'Cleaned up ' . count($jobsToDelete) . ' failed job(s) with no results.'); } @@ -619,18 +622,7 @@ class BulkInfoProviderImportController extends AbstractController */ private function searchLcscBatch(array $keywords): array { - // Get LCSC provider through reflection since PartInfoRetriever doesn't expose it - $reflection = new \ReflectionClass($this->infoRetriever); - $registryProp = $reflection->getProperty('provider_registry'); - $registryProp->setAccessible(true); - $registry = $registryProp->getValue($this->infoRetriever); - - $lcscProvider = $registry->getProviderByKey('lcsc'); - if ($lcscProvider && method_exists($lcscProvider, 'searchByKeywordsBatch')) { - return $lcscProvider->searchByKeywordsBatch($keywords); - } - - return []; + return $this->LCSCProvider->searchByKeywordsBatch($keywords); } #[Route('/job/{jobId}/part/{partId}/mark-completed', name: 'bulk_info_provider_mark_completed', methods: ['POST'])] @@ -710,4 +702,4 @@ class BulkInfoProviderImportController extends AbstractController 'job_completed' => $job->isCompleted() ]); } -} \ No newline at end of file +} diff --git a/src/Services/InfoProviderSystem/Providers/BatchInfoProviderInterface.php b/src/Services/InfoProviderSystem/Providers/BatchInfoProviderInterface.php new file mode 100644 index 00000000..549f117a --- /dev/null +++ b/src/Services/InfoProviderSystem/Providers/BatchInfoProviderInterface.php @@ -0,0 +1,40 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Services\InfoProviderSystem\Providers; + +use App\Services\InfoProviderSystem\DTOs\SearchResultDTO; + +/** + * This interface marks a provider as a info provider which can provide information directly in batch operations + */ +interface BatchInfoProviderInterface extends InfoProviderInterface +{ + /** + * Search for multiple keywords in a single batch operation and return the results, ordered by the keywords. + * This allows for a more efficient search compared to running multiple single searches. + * @param string[] $keywords + * @return array An associative array where the key is the keyword and the value is the search results for that keyword + */ + public function searchByKeywordsBatch(array $keywords): array; +} diff --git a/src/Services/InfoProviderSystem/Providers/LCSCProvider.php b/src/Services/InfoProviderSystem/Providers/LCSCProvider.php index aad5c6b2..53854623 100755 --- a/src/Services/InfoProviderSystem/Providers/LCSCProvider.php +++ b/src/Services/InfoProviderSystem/Providers/LCSCProvider.php @@ -33,7 +33,7 @@ use App\Settings\InfoProviderSystem\LCSCSettings; use Symfony\Component\HttpFoundation\Cookie; use Symfony\Contracts\HttpClient\HttpClientInterface; -class LCSCProvider implements InfoProviderInterface +class LCSCProvider implements BatchInfoProviderInterface { private const ENDPOINT_URL = 'https://wmsc.lcsc.com/ftps/wm';