Increase time limit on batch search and add option to priorities which fields to choose

src/Controller/BulkInfoProviderImportController.php

@@ -54,6 +54,9 @@ class BulkInfoProviderImportController extends AbstractController
     {
         $this->denyAccessUnlessGranted('@info_providers.create_parts');
         
+        // Increase execution time for bulk operations
+        set_time_limit(600); // 10 minutes for large batches
+
         $ids = $request->query->get('ids');
         if (!$ids) {
             $this->addFlash('error', 'No parts selected for bulk import');
@@ -70,6 +73,11 @@ class BulkInfoProviderImportController extends AbstractController
             return $this->redirectToRoute('homepage');
         }
         
+        // Warn about large batches
+        if (count($parts) > 50) {
+            $this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.');
+        }
+
         // Generate field choices
         $fieldChoices = [
             'info_providers.bulk_search.field.mpn' => 'mpn',
@@ -86,7 +94,7 @@ class BulkInfoProviderImportController extends AbstractController
         // Initialize form with useful default mappings
         $initialData = [
             'field_mappings' => [
-                ['field' => 'mpn', 'providers' => []]
+                ['field' => 'mpn', 'providers' => [], 'priority' => 1]
             ],
             'prefetch_details' => false
         ];
@@ -103,6 +111,12 @@ class BulkInfoProviderImportController extends AbstractController
             $fieldMappings = $formData['field_mappings'];
             $prefetchDetails = $formData['prefetch_details'] ?? false;
             
+            // Debug logging
+            $exceptionLogger->info('Form data received', [
+                'prefetch_details' => $prefetchDetails,
+                'prefetch_details_type' => gettype($prefetchDetails)
+            ]);
+
             // Create and save the job
             $job = new BulkInfoProviderImportJob();
             $job->setFieldMappings($fieldMappings);
@@ -123,7 +137,46 @@ class BulkInfoProviderImportController extends AbstractController
             $this->entityManager->flush();
 
             $searchResults = [];
+            $hasAnyResults = false;
 
+            try {
+                // Optimize: Use batch async requests for LCSC provider
+                $lcscKeywords = [];
+                $keywordToPartField = [];
+                
+                // First, collect all LCSC keywords for batch processing
+                foreach ($parts as $part) {
+                    foreach ($fieldMappings as $mapping) {
+                        $field = $mapping['field'];
+                        $providers = $mapping['providers'] ?? [];
+                        
+                        if (in_array('lcsc', $providers, true)) {
+                            $keyword = $this->getKeywordFromField($part, $field);
+                            if ($keyword) {
+                                $lcscKeywords[] = $keyword;
+                                $keywordToPartField[$keyword] = [
+                                    'part' => $part,
+                                    'field' => $field
+                                ];
+                            }
+                        }
+                    }
+                }
+
+                // Batch search LCSC keywords asynchronously
+                $lcscBatchResults = [];
+                if (!empty($lcscKeywords)) {
+                    try {
+                        // Try to get LCSC provider and use batch method if available
+                        $lcscBatchResults = $this->searchLcscBatch($lcscKeywords);
+                    } catch (\Exception $e) {
+                        $exceptionLogger->warning('LCSC batch search failed, falling back to individual requests', [
+                            'error' => $e->getMessage()
+                        ]);
+                    }
+                }
+
+                // Now process each part
                 foreach ($parts as $part) {
                     $partResult = [
                         'part' => $part,
@@ -131,11 +184,24 @@ class BulkInfoProviderImportController extends AbstractController
                         'errors' => []
                     ];
 
-                // Collect all DTOs from all applicable field mappings
+                    // Collect all DTOs using priority-based search
                     $allDtos = [];
                     $dtoMetadata = []; // Store source field info separately
 
+                    // Group mappings by priority (lower number = higher priority)
+                    $mappingsByPriority = [];
                     foreach ($fieldMappings as $mapping) {
+                        $priority = $mapping['priority'] ?? 1;
+                        $mappingsByPriority[$priority][] = $mapping;
+                    }
+                    ksort($mappingsByPriority); // Sort by priority (1, 2, 3...)
+
+                    // Try each priority level until we find results
+                    foreach ($mappingsByPriority as $priority => $mappings) {
+                        $priorityResults = [];
+
+                        // For same priority, search all and combine results
+                        foreach ($mappings as $mapping) {
                             $field = $mapping['field'];
                             $providers = $mapping['providers'] ?? [];
 
@@ -147,28 +213,42 @@ class BulkInfoProviderImportController extends AbstractController
 
                             if ($keyword) {
                                 try {
+                                    // Use batch results for LCSC if available
+                                    if (in_array('lcsc', $providers, true) && isset($lcscBatchResults[$keyword])) {
+                                        $dtos = $lcscBatchResults[$keyword];
+                                    } else {
+                                        // Fall back to regular search for non-LCSC providers
                                         $dtos = $this->infoRetriever->searchByKeyword(
                                             keyword: $keyword,
                                             providers: $providers
                                         );
+                                    }
 
                                     // Store field info for each DTO separately
                                     foreach ($dtos as $dto) {
                                         $dtoKey = $dto->provider_key . '|' . $dto->provider_id;
                                         $dtoMetadata[$dtoKey] = [
                                             'source_field' => $field,
-                                    'source_keyword' => $keyword
+                                            'source_keyword' => $keyword,
+                                            'priority' => $priority
                                         ];
                                     }
 
-                            $allDtos = array_merge($allDtos, $dtos);
+                                    $priorityResults = array_merge($priorityResults, $dtos);
                                 } catch (ClientException $e) {
-                            $partResult['errors'][] = "Error searching with {$field}: " . $e->getMessage();
+                                    $partResult['errors'][] = "Error searching with {$field} (priority {$priority}): " . $e->getMessage();
                                     $exceptionLogger->error('Error during bulk info provider search for part ' . $part->getId() . " field {$field}: " . $e->getMessage(), ['exception' => $e]);
                                 }
                             }
                         }
 
+                        // If we found results at this priority level, use them and stop
+                        if (!empty($priorityResults)) {
+                            $allDtos = $priorityResults;
+                            break;
+                        }
+                    }
+
                     // Remove duplicates based on provider_key + provider_id
                     $uniqueDtos = [];
                     $seenKeys = [];
@@ -198,17 +278,54 @@ class BulkInfoProviderImportController extends AbstractController
                         $uniqueDtos
                     );
 
+                    if (!empty($partResult['search_results'])) {
+                        $hasAnyResults = true;
+                    }
+
                     $searchResults[] = $partResult;
                 }
 
+                // Check if search was successful
+                if (!$hasAnyResults) {
+                    $exceptionLogger->warning('Bulk import search returned no results for any parts', [
+                        'job_id' => $job->getId(),
+                        'parts_count' => count($parts)
+                    ]);
+                    
+                    // Delete the job since it has no useful results
+                    $this->entityManager->remove($job);
+                    $this->entityManager->flush();
+                    
+                    $this->addFlash('error', 'No search results found for any of the selected parts. Please check your field mappings and provider selections.');
+                    return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
+                }
+
                 // Save search results to job
                 $job->setSearchResults($this->serializeSearchResults($searchResults));
                 $job->markAsInProgress();
                 $this->entityManager->flush();
                 
+            } catch (\Exception $e) {
+                $exceptionLogger->error('Critical error during bulk import search', [
+                    'job_id' => $job->getId(),
+                    'error' => $e->getMessage(),
+                    'exception' => $e
+                ]);
+                
+                // Delete the job on critical failure
+                $this->entityManager->remove($job);
+                $this->entityManager->flush();
+                
+                $this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage());
+                return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
+            }
+
             // Prefetch details if requested
             if ($prefetchDetails) {
+                $exceptionLogger->info('Prefetch details requested, starting prefetch for ' . count($searchResults) . ' parts');
                 $this->prefetchDetailsForResults($searchResults, $exceptionLogger);
+            } else {
+                $exceptionLogger->info('Prefetch details not requested, skipping prefetch');
             }
 
             // Redirect to step 2 with the job
@@ -236,21 +353,40 @@ class BulkInfoProviderImportController extends AbstractController
             ->findBy([], ['createdAt' => 'DESC']);
 
         // Check and auto-complete jobs that should be completed
+        // Also clean up jobs with no results (failed searches)
         $updatedJobs = false;
+        $jobsToDelete = [];
+        
         foreach ($allJobs as $job) {
             if ($job->isAllPartsCompleted() && !$job->isCompleted()) {
                 $job->markAsCompleted();
                 $updatedJobs = true;
             }
+            
+            // Mark jobs with no results for deletion (failed searches)
+            if ($job->getResultCount() === 0 && $job->isInProgress()) {
+                $jobsToDelete[] = $job;
+            }
+        }
+        
+        // Delete failed jobs
+        foreach ($jobsToDelete as $job) {
+            $this->entityManager->remove($job);
+            $updatedJobs = true;
         }
 
         // Flush changes if any jobs were updated
         if ($updatedJobs) {
             $this->entityManager->flush();
+            
+            if (!empty($jobsToDelete)) {
+                $this->addFlash('info', 'Cleaned up ' . count($jobsToDelete) . ' failed job(s) with no results.');
+            }
         }
 
         return $this->render('info_providers/bulk_import/manage.html.twig', [
-            'jobs' => $allJobs
+            'jobs' => $this->entityManager->getRepository(BulkInfoProviderImportJob::class)
+                ->findBy([], ['createdAt' => 'DESC']) // Refetch after cleanup
         ]);
     }
 
@@ -478,6 +614,25 @@ class BulkInfoProviderImportController extends AbstractController
         return $searchResults;
     }
 
+    /**
+     * Perform batch LCSC search using async HTTP requests
+     */
+    private function searchLcscBatch(array $keywords): array
+    {
+        // Get LCSC provider through reflection since PartInfoRetriever doesn't expose it
+        $reflection = new \ReflectionClass($this->infoRetriever);
+        $registryProp = $reflection->getProperty('provider_registry');
+        $registryProp->setAccessible(true);
+        $registry = $registryProp->getValue($this->infoRetriever);
+        
+        $lcscProvider = $registry->getProviderByKey('lcsc');
+        if ($lcscProvider && method_exists($lcscProvider, 'searchByKeywordsBatch')) {
+            return $lcscProvider->searchByKeywordsBatch($keywords);
+        }
+        
+        return [];
+    }
+
     #[Route('/job/{jobId}/part/{partId}/mark-completed', name: 'bulk_info_provider_mark_completed', methods: ['POST'])]
     public function markPartCompleted(int $jobId, int $partId): Response
     {

src/Controller/PartController.php

@@ -65,12 +65,14 @@ use function Symfony\Component\Translation\t;
 #[Route(path: '/part')]
 class PartController extends AbstractController
 {
-    public function __construct(protected PricedetailHelper $pricedetailHelper,
+    public function __construct(
+        protected PricedetailHelper $pricedetailHelper,
         protected PartPreviewGenerator $partPreviewGenerator,
         private readonly TranslatorInterface $translator,
-        private readonly AttachmentSubmitHandler $attachmentSubmitHandler, private readonly EntityManagerInterface $em,
+        private readonly AttachmentSubmitHandler $attachmentSubmitHandler,
-        protected EventCommentHelper $commentHelper)
+        private readonly EntityManagerInterface $em,
-    {
+        protected EventCommentHelper $commentHelper
+    ) {
     }
 
     /**
@@ -79,9 +81,16 @@ class PartController extends AbstractController
      */
     #[Route(path: '/{id}/info/{timestamp}', name: 'part_info')]
     #[Route(path: '/{id}', requirements: ['id' => '\d+'])]
-    public function show(Part $part, Request $request, TimeTravel $timeTravel, HistoryHelper $historyHelper,
+    public function show(
-        DataTableFactory $dataTable, ParameterExtractor $parameterExtractor, PartLotWithdrawAddHelper $withdrawAddHelper, ?string $timestamp = null): Response
+        Part $part,
-    {
+        Request $request,
+        TimeTravel $timeTravel,
+        HistoryHelper $historyHelper,
+        DataTableFactory $dataTable,
+        ParameterExtractor $parameterExtractor,
+        PartLotWithdrawAddHelper $withdrawAddHelper,
+        ?string $timestamp = null
+    ): Response {
         $this->denyAccessUnlessGranted('read', $part);
 
         $timeTravel_timestamp = null;
@@ -194,11 +203,15 @@ class PartController extends AbstractController
     #[Route(path: '/new', name: 'part_new')]
     #[Route(path: '/{id}/clone', name: 'part_clone')]
     #[Route(path: '/new_build_part/{project_id}', name: 'part_new_build_part')]
-    public function new(Request $request, EntityManagerInterface $em, TranslatorInterface $translator,
+    public function new(
-        AttachmentSubmitHandler $attachmentSubmitHandler, ProjectBuildPartHelper $projectBuildPartHelper,
+        Request $request,
+        EntityManagerInterface $em,
+        TranslatorInterface $translator,
+        AttachmentSubmitHandler $attachmentSubmitHandler,
+        ProjectBuildPartHelper $projectBuildPartHelper,
         #[MapEntity(mapping: ['id' => 'id'])] ?Part $part = null,
-        #[MapEntity(mapping: ['project_id' => 'id'])] ?Project $project = null): Response
+        #[MapEntity(mapping: ['project_id' => 'id'])] ?Project $project = null
-    {
+    ): Response {
 
         if ($part instanceof Part) {
             //Clone part
@@ -293,9 +306,14 @@ class PartController extends AbstractController
     }
 
     #[Route(path: '/{id}/from_info_provider/{providerKey}/{providerId}/update', name: 'info_providers_update_part', requirements: ['providerId' => '.+'])]
-    public function updateFromInfoProvider(Part $part, Request $request, string $providerKey, string $providerId,
+    public function updateFromInfoProvider(
-        PartInfoRetriever $infoRetriever, PartMerger $partMerger): Response
+        Part $part,
-    {
+        Request $request,
+        string $providerKey,
+        string $providerId,
+        PartInfoRetriever $infoRetriever,
+        PartMerger $partMerger
+    ): Response {
         $this->denyAccessUnlessGranted('edit', $part);
         $this->denyAccessUnlessGranted('@info_providers.create_parts');
 
@@ -424,7 +442,8 @@ class PartController extends AbstractController
             $template = 'parts/edit/update_from_ip.html.twig';
         }
 
-        return $this->render($template,
+        return $this->render(
+            $template,
             [
                 'part' => $new_part,
                 'form' => $form,
@@ -432,7 +451,8 @@ class PartController extends AbstractController
                 'merge_other' => $merge_infos['other_part'] ?? null,
                 'bulk_job' => $merge_infos['bulk_job'] ?? null,
                 'jobId' => $request->query->get('jobId')
-            ]);
+            ]
+        );
     }
 
 

src/Form/InfoProviderSystem/FieldToProviderMappingType.php

@@ -24,6 +24,7 @@ namespace App\Form\InfoProviderSystem;
 
 use Symfony\Component\Form\AbstractType;
 use Symfony\Component\Form\Extension\Core\Type\ChoiceType;
+use Symfony\Component\Form\Extension\Core\Type\IntegerType;
 use Symfony\Component\Form\FormBuilderInterface;
 use Symfony\Component\OptionsResolver\OptionsResolver;
 
@@ -47,6 +48,19 @@ class FieldToProviderMappingType extends AbstractType
             'help' => 'info_providers.bulk_search.providers.help',
             'required' => false,
         ]);
+
+        $builder->add('priority', IntegerType::class, [
+            'label' => 'info_providers.bulk_search.priority',
+            'help' => 'info_providers.bulk_search.priority.help',
+            'required' => false,
+            'data' => 1, // Default priority
+            'attr' => [
+                'min' => 1,
+                'max' => 10,
+                'class' => 'form-control-sm',
+                'style' => 'width: 80px;'
+            ]
+        ]);
     }
 
     public function configureOptions(OptionsResolver $resolver): void

templates/info_providers/bulk_import/step1.html.twig

@@ -31,7 +31,7 @@
                                 <th>{% trans %}info_providers.bulk_import.progress{% endtrans %}</th>
                                 <th>{% trans %}info_providers.bulk_import.status{% endtrans %}</th>
                                 <th>{% trans %}info_providers.bulk_import.created_at{% endtrans %}</th>
-                                <th>{% trans %}action.label{% endtrans %}</th>
+                                <th>{% trans %}info_providers.bulk_import.action.label{% endtrans %}</th>
                             </tr>
                         </thead>
                         <tbody>
@@ -87,6 +87,14 @@
         {% trans %}info_providers.bulk_import.step1.global_mapping_description{% endtrans %}
     </div>
     
+    <div class="alert alert-success" role="alert">
+        <i class="fas fa-lightbulb"></i>
+        <strong>{% trans %}info_providers.bulk_import.priority_system.title{% endtrans %}:</strong> {% trans %}info_providers.bulk_import.priority_system.description{% endtrans %}
+        <br><small class="text-muted">
+            {% trans %}info_providers.bulk_import.priority_system.example{% endtrans %}
+        </small>
+    </div>
+    
     <div class="alert alert-warning" role="alert">
         <i class="fas fa-exclamation-triangle"></i>
         {% trans %}info_providers.bulk_import.step1.spn_recommendation{% endtrans %}
@@ -138,6 +146,7 @@
                         <tr>
                             <th>{% trans %}info_providers.bulk_search.search_field{% endtrans %}</th>
                             <th>{% trans %}info_providers.bulk_search.providers{% endtrans %}</th>
+                            <th width="80">{% trans %}info_providers.bulk_search.priority{% endtrans %}</th>
                             <th width="100">{% trans %}info_providers.bulk_import.actions.label{% endtrans %}</th>
                         </tr>
                     </thead>
@@ -146,6 +155,7 @@
                             <tr class="mapping-row">
                                 <td>{{ form_widget(mapping.field) }}{{ form_errors(mapping.field) }}</td>
                                 <td>{{ form_widget(mapping.providers) }}{{ form_errors(mapping.providers) }}</td>
+                                <td>{{ form_widget(mapping.priority) }}{{ form_errors(mapping.priority) }}</td>
                                 <td>
                                     <button type="button" class="btn btn-danger btn-sm" onclick="removeMapping(this)">
                                         <i class="fas fa-trash"></i>
@@ -217,7 +227,7 @@
                                         <th>{% trans %}manufacturer.label{% endtrans %}</th>
                                         <th>{% trans %}info_providers.table.provider.label{% endtrans %}</th>
                                         <th>{% trans %}info_providers.bulk_import.source_field{% endtrans %}</th>
-                                        <th>{% trans %}action.label{% endtrans %}</th>
+                                        <th>{% trans %}info_providers.bulk_import.action.label{% endtrans %}</th>
                                     </tr>
                                 </thead>
                                 <tbody>
@@ -307,9 +317,10 @@ function addMapping() {
     const tempDiv = document.createElement('div');
     tempDiv.innerHTML = newRowHtml;
     
-    // Extract field and provider widgets
+    // Extract field, provider, and priority widgets
     const fieldWidget = tempDiv.querySelector('select[name*="[field]"]') || tempDiv.children[0];
     const providerWidget = tempDiv.querySelector('select[name*="[providers]"]') || tempDiv.children[1];
+    const priorityWidget = tempDiv.querySelector('input[name*="[priority]"]') || tempDiv.children[2];
     
     // Create new row
     const newRow = document.createElement('tr');
@@ -317,6 +328,7 @@ function addMapping() {
     newRow.innerHTML = `
         <td>${fieldWidget ? fieldWidget.outerHTML : ''}</td>
         <td>${providerWidget ? providerWidget.outerHTML : ''}</td>
+        <td>${priorityWidget ? priorityWidget.outerHTML : ''}</td>
         <td>
             <button type="button" class="btn btn-danger btn-sm" onclick="removeMapping(this)">
                 <i class="fas fa-trash"></i>

templates/info_providers/bulk_import/step2.html.twig

@@ -140,7 +140,7 @@
                                     <th>{% trans %}manufacturer.label{% endtrans %}</th>
                                     <th>{% trans %}info_providers.table.provider.label{% endtrans %}</th>
                                     <th>{% trans %}info_providers.bulk_import.source_field{% endtrans %}</th>
-                                    <th>{% trans %}action.label{% endtrans %}</th>
+                                    <th>{% trans %}info_providers.bulk_import.action.label{% endtrans %}</th>
                                 </tr>
                             </thead>
                             <tbody>
@@ -178,7 +178,7 @@
                                             <div class="btn-group-vertical btn-group-sm" role="group">
                                                 {% set updateHref = path('info_providers_update_part',
                                                     {'id': part.id, 'providerKey': dto.provider_key, 'providerId': dto.provider_id}) ~ '?jobId=' ~ job.id %}
-                                                <a class="btn btn-primary{% if isCompleted %} disabled{% endif %}" href="{% if not isCompleted %}{{ updateHref }}{% else %}#{% endif %}" target="_blank"{% if isCompleted %} aria-disabled="true"{% endif %}>
+                                                <a class="btn btn-primary{% if isCompleted %} disabled{% endif %}" href="{% if not isCompleted %}{{ updateHref }}{% else %}#{% endif %}"{% if isCompleted %} aria-disabled="true"{% endif %}>
                                                     <i class="fas fa-edit"></i> {% trans %}info_providers.bulk_import.update_part{% endtrans %}
                                                 </a>
                                             </div>

translations/messages.en.xlf

@@ -13861,5 +13861,35 @@ Please note, that you can not impersonate a disabled user. If you try you will g
         <target>Bulk Info Provider Import</target>
       </segment>
     </unit>
+    <unit id="XQV7AkF" name="info_providers.bulk_search.priority">
+      <segment state="translated">
+        <source>info_providers.bulk_search.priority</source>
+        <target>Priority</target>
+      </segment>
+    </unit>
+    <unit id="XR9mZP_" name="info_providers.bulk_search.priority.help">
+      <segment state="translated">
+        <source>info_providers.bulk_search.priority.help</source>
+        <target>Lower numbers = higher priority. Same priority = combine results. Different priorities = try highest first, fallback if no results.</target>
+      </segment>
+    </unit>
+    <unit id="Iz3Ow0_" name="info_providers.bulk_import.priority_system.title">
+      <segment state="translated">
+        <source>info_providers.bulk_import.priority_system.title</source>
+        <target>Priority System</target>
+      </segment>
+    </unit>
+    <unit id="CPwXFjE" name="info_providers.bulk_import.priority_system.description">
+      <segment state="translated">
+        <source>info_providers.bulk_import.priority_system.description</source>
+        <target>Lower numbers = higher priority. Same priority = combine results. Different priorities = try highest first, fallback if no results.</target>
+      </segment>
+    </unit>
+    <unit id="v.rTI5s" name="info_providers.bulk_import.priority_system.example">
+      <segment state="translated">
+        <source>info_providers.bulk_import.priority_system.example</source>
+        <target>Example: Priority 1: "LCSC SPN → LCSC", Priority 2: "MPN → LCSC + Mouser", Priority 3: "Name → All providers"</target>
+      </segment>
+    </unit>
   </file>
 </xliff>