diff --git a/src/Controller/BulkInfoProviderImportController.php b/src/Controller/BulkInfoProviderImportController.php index 6c434191..d09e8d04 100644 --- a/src/Controller/BulkInfoProviderImportController.php +++ b/src/Controller/BulkInfoProviderImportController.php @@ -53,6 +53,9 @@ class BulkInfoProviderImportController extends AbstractController public function step1(Request $request, LoggerInterface $exceptionLogger): Response { $this->denyAccessUnlessGranted('@info_providers.create_parts'); + + // Increase execution time for bulk operations + set_time_limit(600); // 10 minutes for large batches $ids = $request->query->get('ids'); if (!$ids) { @@ -69,6 +72,11 @@ class BulkInfoProviderImportController extends AbstractController $this->addFlash('error', 'No valid parts found for bulk import'); return $this->redirectToRoute('homepage'); } + + // Warn about large batches + if (count($parts) > 50) { + $this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.'); + } // Generate field choices $fieldChoices = [ @@ -86,7 +94,7 @@ class BulkInfoProviderImportController extends AbstractController // Initialize form with useful default mappings $initialData = [ 'field_mappings' => [ - ['field' => 'mpn', 'providers' => []] + ['field' => 'mpn', 'providers' => [], 'priority' => 1] ], 'prefetch_details' => false ]; @@ -102,6 +110,12 @@ class BulkInfoProviderImportController extends AbstractController $formData = $form->getData(); $fieldMappings = $formData['field_mappings']; $prefetchDetails = $formData['prefetch_details'] ?? false; + + // Debug logging + $exceptionLogger->info('Form data received', [ + 'prefetch_details' => $prefetchDetails, + 'prefetch_details_type' => gettype($prefetchDetails) + ]); // Create and save the job $job = new BulkInfoProviderImportJob(); @@ -123,92 +137,195 @@ class BulkInfoProviderImportController extends AbstractController $this->entityManager->flush(); $searchResults = []; + $hasAnyResults = false; - foreach ($parts as $part) { - $partResult = [ - 'part' => $part, - 'search_results' => [], - 'errors' => [] - ]; - - // Collect all DTOs from all applicable field mappings - $allDtos = []; - $dtoMetadata = []; // Store source field info separately - - foreach ($fieldMappings as $mapping) { - $field = $mapping['field']; - $providers = $mapping['providers'] ?? []; - - if (empty($providers)) { - continue; - } - - $keyword = $this->getKeywordFromField($part, $field); - - if ($keyword) { - try { - $dtos = $this->infoRetriever->searchByKeyword( - keyword: $keyword, - providers: $providers - ); - - // Store field info for each DTO separately - foreach ($dtos as $dto) { - $dtoKey = $dto->provider_key . '|' . $dto->provider_id; - $dtoMetadata[$dtoKey] = [ - 'source_field' => $field, - 'source_keyword' => $keyword + try { + // Optimize: Use batch async requests for LCSC provider + $lcscKeywords = []; + $keywordToPartField = []; + + // First, collect all LCSC keywords for batch processing + foreach ($parts as $part) { + foreach ($fieldMappings as $mapping) { + $field = $mapping['field']; + $providers = $mapping['providers'] ?? []; + + if (in_array('lcsc', $providers, true)) { + $keyword = $this->getKeywordFromField($part, $field); + if ($keyword) { + $lcscKeywords[] = $keyword; + $keywordToPartField[$keyword] = [ + 'part' => $part, + 'field' => $field ]; } - - $allDtos = array_merge($allDtos, $dtos); - } catch (ClientException $e) { - $partResult['errors'][] = "Error searching with {$field}: " . $e->getMessage(); - $exceptionLogger->error('Error during bulk info provider search for part ' . $part->getId() . " field {$field}: " . $e->getMessage(), ['exception' => $e]); } } } - // Remove duplicates based on provider_key + provider_id - $uniqueDtos = []; - $seenKeys = []; - foreach ($allDtos as $dto) { - if ($dto === null || !isset($dto->provider_key, $dto->provider_id)) { - continue; - } - $key = "{$dto->provider_key}|{$dto->provider_id}"; - if (!in_array($key, $seenKeys, true)) { - $seenKeys[] = $key; - $uniqueDtos[] = $dto; + // Batch search LCSC keywords asynchronously + $lcscBatchResults = []; + if (!empty($lcscKeywords)) { + try { + // Try to get LCSC provider and use batch method if available + $lcscBatchResults = $this->searchLcscBatch($lcscKeywords); + } catch (\Exception $e) { + $exceptionLogger->warning('LCSC batch search failed, falling back to individual requests', [ + 'error' => $e->getMessage() + ]); } } - // Convert DTOs to result format with metadata - $partResult['search_results'] = array_map( - function ($dto) use ($dtoMetadata) { - $dtoKey = $dto->provider_key . '|' . $dto->provider_id; - $metadata = $dtoMetadata[$dtoKey] ?? []; - return [ - 'dto' => $dto, - 'localPart' => $this->existingPartFinder->findFirstExisting($dto), - 'source_field' => $metadata['source_field'] ?? null, - 'source_keyword' => $metadata['source_keyword'] ?? null - ]; - }, - $uniqueDtos - ); + // Now process each part + foreach ($parts as $part) { + $partResult = [ + 'part' => $part, + 'search_results' => [], + 'errors' => [] + ]; - $searchResults[] = $partResult; + // Collect all DTOs using priority-based search + $allDtos = []; + $dtoMetadata = []; // Store source field info separately + + // Group mappings by priority (lower number = higher priority) + $mappingsByPriority = []; + foreach ($fieldMappings as $mapping) { + $priority = $mapping['priority'] ?? 1; + $mappingsByPriority[$priority][] = $mapping; + } + ksort($mappingsByPriority); // Sort by priority (1, 2, 3...) + + // Try each priority level until we find results + foreach ($mappingsByPriority as $priority => $mappings) { + $priorityResults = []; + + // For same priority, search all and combine results + foreach ($mappings as $mapping) { + $field = $mapping['field']; + $providers = $mapping['providers'] ?? []; + + if (empty($providers)) { + continue; + } + + $keyword = $this->getKeywordFromField($part, $field); + + if ($keyword) { + try { + // Use batch results for LCSC if available + if (in_array('lcsc', $providers, true) && isset($lcscBatchResults[$keyword])) { + $dtos = $lcscBatchResults[$keyword]; + } else { + // Fall back to regular search for non-LCSC providers + $dtos = $this->infoRetriever->searchByKeyword( + keyword: $keyword, + providers: $providers + ); + } + + // Store field info for each DTO separately + foreach ($dtos as $dto) { + $dtoKey = $dto->provider_key . '|' . $dto->provider_id; + $dtoMetadata[$dtoKey] = [ + 'source_field' => $field, + 'source_keyword' => $keyword, + 'priority' => $priority + ]; + } + + $priorityResults = array_merge($priorityResults, $dtos); + } catch (ClientException $e) { + $partResult['errors'][] = "Error searching with {$field} (priority {$priority}): " . $e->getMessage(); + $exceptionLogger->error('Error during bulk info provider search for part ' . $part->getId() . " field {$field}: " . $e->getMessage(), ['exception' => $e]); + } + } + } + + // If we found results at this priority level, use them and stop + if (!empty($priorityResults)) { + $allDtos = $priorityResults; + break; + } + } + + // Remove duplicates based on provider_key + provider_id + $uniqueDtos = []; + $seenKeys = []; + foreach ($allDtos as $dto) { + if ($dto === null || !isset($dto->provider_key, $dto->provider_id)) { + continue; + } + $key = "{$dto->provider_key}|{$dto->provider_id}"; + if (!in_array($key, $seenKeys, true)) { + $seenKeys[] = $key; + $uniqueDtos[] = $dto; + } + } + + // Convert DTOs to result format with metadata + $partResult['search_results'] = array_map( + function ($dto) use ($dtoMetadata) { + $dtoKey = $dto->provider_key . '|' . $dto->provider_id; + $metadata = $dtoMetadata[$dtoKey] ?? []; + return [ + 'dto' => $dto, + 'localPart' => $this->existingPartFinder->findFirstExisting($dto), + 'source_field' => $metadata['source_field'] ?? null, + 'source_keyword' => $metadata['source_keyword'] ?? null + ]; + }, + $uniqueDtos + ); + + if (!empty($partResult['search_results'])) { + $hasAnyResults = true; + } + + $searchResults[] = $partResult; + } + + // Check if search was successful + if (!$hasAnyResults) { + $exceptionLogger->warning('Bulk import search returned no results for any parts', [ + 'job_id' => $job->getId(), + 'parts_count' => count($parts) + ]); + + // Delete the job since it has no useful results + $this->entityManager->remove($job); + $this->entityManager->flush(); + + $this->addFlash('error', 'No search results found for any of the selected parts. Please check your field mappings and provider selections.'); + return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]); + } + + // Save search results to job + $job->setSearchResults($this->serializeSearchResults($searchResults)); + $job->markAsInProgress(); + $this->entityManager->flush(); + + } catch (\Exception $e) { + $exceptionLogger->error('Critical error during bulk import search', [ + 'job_id' => $job->getId(), + 'error' => $e->getMessage(), + 'exception' => $e + ]); + + // Delete the job on critical failure + $this->entityManager->remove($job); + $this->entityManager->flush(); + + $this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage()); + return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]); } - // Save search results to job - $job->setSearchResults($this->serializeSearchResults($searchResults)); - $job->markAsInProgress(); - $this->entityManager->flush(); - // Prefetch details if requested if ($prefetchDetails) { + $exceptionLogger->info('Prefetch details requested, starting prefetch for ' . count($searchResults) . ' parts'); $this->prefetchDetailsForResults($searchResults, $exceptionLogger); + } else { + $exceptionLogger->info('Prefetch details not requested, skipping prefetch'); } // Redirect to step 2 with the job @@ -236,21 +353,40 @@ class BulkInfoProviderImportController extends AbstractController ->findBy([], ['createdAt' => 'DESC']); // Check and auto-complete jobs that should be completed + // Also clean up jobs with no results (failed searches) $updatedJobs = false; + $jobsToDelete = []; + foreach ($allJobs as $job) { if ($job->isAllPartsCompleted() && !$job->isCompleted()) { $job->markAsCompleted(); $updatedJobs = true; } + + // Mark jobs with no results for deletion (failed searches) + if ($job->getResultCount() === 0 && $job->isInProgress()) { + $jobsToDelete[] = $job; + } + } + + // Delete failed jobs + foreach ($jobsToDelete as $job) { + $this->entityManager->remove($job); + $updatedJobs = true; } // Flush changes if any jobs were updated if ($updatedJobs) { $this->entityManager->flush(); + + if (!empty($jobsToDelete)) { + $this->addFlash('info', 'Cleaned up ' . count($jobsToDelete) . ' failed job(s) with no results.'); + } } return $this->render('info_providers/bulk_import/manage.html.twig', [ - 'jobs' => $allJobs + 'jobs' => $this->entityManager->getRepository(BulkInfoProviderImportJob::class) + ->findBy([], ['createdAt' => 'DESC']) // Refetch after cleanup ]); } @@ -478,6 +614,25 @@ class BulkInfoProviderImportController extends AbstractController return $searchResults; } + /** + * Perform batch LCSC search using async HTTP requests + */ + private function searchLcscBatch(array $keywords): array + { + // Get LCSC provider through reflection since PartInfoRetriever doesn't expose it + $reflection = new \ReflectionClass($this->infoRetriever); + $registryProp = $reflection->getProperty('provider_registry'); + $registryProp->setAccessible(true); + $registry = $registryProp->getValue($this->infoRetriever); + + $lcscProvider = $registry->getProviderByKey('lcsc'); + if ($lcscProvider && method_exists($lcscProvider, 'searchByKeywordsBatch')) { + return $lcscProvider->searchByKeywordsBatch($keywords); + } + + return []; + } + #[Route('/job/{jobId}/part/{partId}/mark-completed', name: 'bulk_info_provider_mark_completed', methods: ['POST'])] public function markPartCompleted(int $jobId, int $partId): Response { diff --git a/src/Controller/PartController.php b/src/Controller/PartController.php index e9c577f0..d1087254 100644 --- a/src/Controller/PartController.php +++ b/src/Controller/PartController.php @@ -65,12 +65,14 @@ use function Symfony\Component\Translation\t; #[Route(path: '/part')] class PartController extends AbstractController { - public function __construct(protected PricedetailHelper $pricedetailHelper, + public function __construct( + protected PricedetailHelper $pricedetailHelper, protected PartPreviewGenerator $partPreviewGenerator, private readonly TranslatorInterface $translator, - private readonly AttachmentSubmitHandler $attachmentSubmitHandler, private readonly EntityManagerInterface $em, - protected EventCommentHelper $commentHelper) - { + private readonly AttachmentSubmitHandler $attachmentSubmitHandler, + private readonly EntityManagerInterface $em, + protected EventCommentHelper $commentHelper + ) { } /** @@ -79,9 +81,16 @@ class PartController extends AbstractController */ #[Route(path: '/{id}/info/{timestamp}', name: 'part_info')] #[Route(path: '/{id}', requirements: ['id' => '\d+'])] - public function show(Part $part, Request $request, TimeTravel $timeTravel, HistoryHelper $historyHelper, - DataTableFactory $dataTable, ParameterExtractor $parameterExtractor, PartLotWithdrawAddHelper $withdrawAddHelper, ?string $timestamp = null): Response - { + public function show( + Part $part, + Request $request, + TimeTravel $timeTravel, + HistoryHelper $historyHelper, + DataTableFactory $dataTable, + ParameterExtractor $parameterExtractor, + PartLotWithdrawAddHelper $withdrawAddHelper, + ?string $timestamp = null + ): Response { $this->denyAccessUnlessGranted('read', $part); $timeTravel_timestamp = null; @@ -151,22 +160,22 @@ class PartController extends AbstractController public function markBulkImportComplete(Part $part, int $jobId, Request $request): Response { $this->denyAccessUnlessGranted('edit', $part); - + if (!$this->isCsrfTokenValid('bulk_complete_' . $part->getId(), $request->request->get('_token'))) { throw $this->createAccessDeniedException('Invalid CSRF token'); } - + $bulkJob = $this->em->getRepository(\App\Entity\BulkInfoProviderImportJob::class)->find($jobId); if (!$bulkJob || $bulkJob->getCreatedBy() !== $this->getUser()) { throw $this->createNotFoundException('Bulk import job not found'); } - + $bulkJob->markPartAsCompleted($part->getId()); $this->em->persist($bulkJob); $this->em->flush(); - + $this->addFlash('success', 'Part marked as completed in bulk import'); - + return $this->redirectToRoute('bulk_info_provider_step2', ['jobId' => $jobId]); } @@ -175,7 +184,7 @@ class PartController extends AbstractController { $this->denyAccessUnlessGranted('delete', $part); - if ($this->isCsrfTokenValid('delete'.$part->getID(), $request->request->get('_token'))) { + if ($this->isCsrfTokenValid('delete' . $part->getID(), $request->request->get('_token'))) { $this->commentHelper->setMessage($request->request->get('log_comment', null)); @@ -194,11 +203,15 @@ class PartController extends AbstractController #[Route(path: '/new', name: 'part_new')] #[Route(path: '/{id}/clone', name: 'part_clone')] #[Route(path: '/new_build_part/{project_id}', name: 'part_new_build_part')] - public function new(Request $request, EntityManagerInterface $em, TranslatorInterface $translator, - AttachmentSubmitHandler $attachmentSubmitHandler, ProjectBuildPartHelper $projectBuildPartHelper, + public function new( + Request $request, + EntityManagerInterface $em, + TranslatorInterface $translator, + AttachmentSubmitHandler $attachmentSubmitHandler, + ProjectBuildPartHelper $projectBuildPartHelper, #[MapEntity(mapping: ['id' => 'id'])] ?Part $part = null, - #[MapEntity(mapping: ['project_id' => 'id'])] ?Project $project = null): Response - { + #[MapEntity(mapping: ['project_id' => 'id'])] ?Project $project = null + ): Response { if ($part instanceof Part) { //Clone part @@ -293,9 +306,14 @@ class PartController extends AbstractController } #[Route(path: '/{id}/from_info_provider/{providerKey}/{providerId}/update', name: 'info_providers_update_part', requirements: ['providerId' => '.+'])] - public function updateFromInfoProvider(Part $part, Request $request, string $providerKey, string $providerId, - PartInfoRetriever $infoRetriever, PartMerger $partMerger): Response - { + public function updateFromInfoProvider( + Part $part, + Request $request, + string $providerKey, + string $providerId, + PartInfoRetriever $infoRetriever, + PartMerger $partMerger + ): Response { $this->denyAccessUnlessGranted('edit', $part); $this->denyAccessUnlessGranted('@info_providers.create_parts'); @@ -359,7 +377,7 @@ class PartController extends AbstractController } catch (AttachmentDownloadException $attachmentDownloadException) { $this->addFlash( 'error', - $this->translator->trans('attachment.download_failed').' '.$attachmentDownloadException->getMessage() + $this->translator->trans('attachment.download_failed') . ' ' . $attachmentDownloadException->getMessage() ); } } @@ -405,7 +423,7 @@ class PartController extends AbstractController if ($jobId && isset($merge_infos['bulk_job'])) { return $this->redirectToRoute('part_edit', ['id' => $new_part->getID(), 'jobId' => $jobId]); } - + return $this->redirectToRoute('part_edit', ['id' => $new_part->getID()]); } @@ -424,7 +442,8 @@ class PartController extends AbstractController $template = 'parts/edit/update_from_ip.html.twig'; } - return $this->render($template, + return $this->render( + $template, [ 'part' => $new_part, 'form' => $form, @@ -432,7 +451,8 @@ class PartController extends AbstractController 'merge_other' => $merge_infos['other_part'] ?? null, 'bulk_job' => $merge_infos['bulk_job'] ?? null, 'jobId' => $request->query->get('jobId') - ]); + ] + ); } @@ -442,17 +462,17 @@ class PartController extends AbstractController if ($this->isCsrfTokenValid('part_withraw' . $part->getID(), $request->request->get('_csfr'))) { //Retrieve partlot from the request $partLot = $em->find(PartLot::class, $request->request->get('lot_id')); - if(!$partLot instanceof PartLot) { + if (!$partLot instanceof PartLot) { throw new \RuntimeException('Part lot not found!'); } //Ensure that the partlot belongs to the part - if($partLot->getPart() !== $part) { + if ($partLot->getPart() !== $part) { throw new \RuntimeException("The origin partlot does not belong to the part!"); } //Try to determine the target lot (used for move actions), if the parameter is existing $targetId = $request->request->get('target_id', null); - $targetLot = $targetId ? $em->find(PartLot::class, $targetId) : null; + $targetLot = $targetId ? $em->find(PartLot::class, $targetId) : null; if ($targetLot && $targetLot->getPart() !== $part) { throw new \RuntimeException("The target partlot does not belong to the part!"); } @@ -466,12 +486,12 @@ class PartController extends AbstractController $timestamp = null; $timestamp_str = $request->request->getString('timestamp', ''); //Try to parse the timestamp - if($timestamp_str !== '') { + if ($timestamp_str !== '') { $timestamp = new DateTime($timestamp_str); } //Ensure that the timestamp is not in the future - if($timestamp !== null && $timestamp > new DateTime("+20min")) { + if ($timestamp !== null && $timestamp > new DateTime("+20min")) { throw new \LogicException("The timestamp must not be in the future!"); } @@ -515,7 +535,7 @@ class PartController extends AbstractController err: //If a redirect was passed, then redirect there - if($request->request->get('_redirect')) { + if ($request->request->get('_redirect')) { return $this->redirect($request->request->get('_redirect')); } //Otherwise just redirect to the part page diff --git a/src/Form/InfoProviderSystem/FieldToProviderMappingType.php b/src/Form/InfoProviderSystem/FieldToProviderMappingType.php index 20506fc8..fa7ee28b 100644 --- a/src/Form/InfoProviderSystem/FieldToProviderMappingType.php +++ b/src/Form/InfoProviderSystem/FieldToProviderMappingType.php @@ -24,6 +24,7 @@ namespace App\Form\InfoProviderSystem; use Symfony\Component\Form\AbstractType; use Symfony\Component\Form\Extension\Core\Type\ChoiceType; +use Symfony\Component\Form\Extension\Core\Type\IntegerType; use Symfony\Component\Form\FormBuilderInterface; use Symfony\Component\OptionsResolver\OptionsResolver; @@ -47,6 +48,19 @@ class FieldToProviderMappingType extends AbstractType 'help' => 'info_providers.bulk_search.providers.help', 'required' => false, ]); + + $builder->add('priority', IntegerType::class, [ + 'label' => 'info_providers.bulk_search.priority', + 'help' => 'info_providers.bulk_search.priority.help', + 'required' => false, + 'data' => 1, // Default priority + 'attr' => [ + 'min' => 1, + 'max' => 10, + 'class' => 'form-control-sm', + 'style' => 'width: 80px;' + ] + ]); } public function configureOptions(OptionsResolver $resolver): void diff --git a/templates/info_providers/bulk_import/step1.html.twig b/templates/info_providers/bulk_import/step1.html.twig index 5c3436de..af6a2fcb 100644 --- a/templates/info_providers/bulk_import/step1.html.twig +++ b/templates/info_providers/bulk_import/step1.html.twig @@ -31,7 +31,7 @@ {% trans %}info_providers.bulk_import.progress{% endtrans %} {% trans %}info_providers.bulk_import.status{% endtrans %} {% trans %}info_providers.bulk_import.created_at{% endtrans %} - {% trans %}action.label{% endtrans %} + {% trans %}info_providers.bulk_import.action.label{% endtrans %} @@ -87,6 +87,14 @@ {% trans %}info_providers.bulk_import.step1.global_mapping_description{% endtrans %} + +