diff --git a/assets/controllers/bulk_import_controller.js b/assets/controllers/bulk_import_controller.js new file mode 100644 index 00000000..49e4d60f --- /dev/null +++ b/assets/controllers/bulk_import_controller.js @@ -0,0 +1,359 @@ +import { Controller } from "@hotwired/stimulus" +import { generateCsrfHeaders } from "./csrf_protection_controller" + +export default class extends Controller { + static targets = ["progressBar", "progressText"] + static values = { + jobId: Number, + partId: Number, + researchUrl: String, + researchAllUrl: String, + markCompletedUrl: String, + markSkippedUrl: String, + markPendingUrl: String + } + + connect() { + // Auto-refresh progress if job is in progress + if (this.hasProgressBarTarget) { + this.startProgressUpdates() + } + + // Restore scroll position after page reload (if any) + this.restoreScrollPosition() + } + + getHeaders() { + const headers = { + 'Content-Type': 'application/json', + 'X-Requested-With': 'XMLHttpRequest' + } + + // Add CSRF headers if available + const form = document.querySelector('form') + if (form) { + const csrfHeaders = generateCsrfHeaders(form) + Object.assign(headers, csrfHeaders) + } + + return headers + } + + async fetchWithErrorHandling(url, options = {}, timeout = 30000) { + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), timeout) + + try { + const response = await fetch(url, { + ...options, + headers: { ...this.getHeaders(), ...options.headers }, + signal: controller.signal + }) + + clearTimeout(timeoutId) + + if (!response.ok) { + const errorText = await response.text() + throw new Error(`Server error (${response.status}): ${errorText}`) + } + + return await response.json() + } catch (error) { + clearTimeout(timeoutId) + + if (error.name === 'AbortError') { + throw new Error('Request timed out. Please try again.') + } else if (error.message.includes('Failed to fetch')) { + throw new Error('Network error. Please check your connection and try again.') + } else { + throw error + } + } + } + + disconnect() { + if (this.progressInterval) { + clearInterval(this.progressInterval) + } + } + + startProgressUpdates() { + // Progress updates are handled via page reload for better reliability + // No need for periodic updates since state changes trigger page refresh + } + + restoreScrollPosition() { + const savedPosition = sessionStorage.getItem('bulkImportScrollPosition') + if (savedPosition) { + // Restore scroll position after a small delay to ensure page is fully loaded + setTimeout(() => { + window.scrollTo(0, parseInt(savedPosition)) + // Clear the saved position so it doesn't interfere with normal navigation + sessionStorage.removeItem('bulkImportScrollPosition') + }, 100) + } + } + + async markCompleted(event) { + const partId = event.currentTarget.dataset.partId + + try { + const url = this.markCompletedUrlValue.replace('__PART_ID__', partId) + const data = await this.fetchWithErrorHandling(url, { method: 'POST' }) + + if (data.success) { + this.updateProgressDisplay(data) + this.markRowAsCompleted(partId) + + if (data.job_completed) { + this.showJobCompletedMessage() + } + } else { + this.showErrorMessage(data.error || 'Failed to mark part as completed') + } + } catch (error) { + console.error('Error marking part as completed:', error) + this.showErrorMessage(error.message || 'Failed to mark part as completed') + } + } + + async markSkipped(event) { + const partId = event.currentTarget.dataset.partId + const reason = prompt('Reason for skipping (optional):') || '' + + try { + const url = this.markSkippedUrlValue.replace('__PART_ID__', partId) + const data = await this.fetchWithErrorHandling(url, { + method: 'POST', + body: JSON.stringify({ reason }) + }) + + if (data.success) { + this.updateProgressDisplay(data) + this.markRowAsSkipped(partId) + } else { + this.showErrorMessage(data.error || 'Failed to mark part as skipped') + } + } catch (error) { + console.error('Error marking part as skipped:', error) + this.showErrorMessage(error.message || 'Failed to mark part as skipped') + } + } + + async markPending(event) { + const partId = event.currentTarget.dataset.partId + + try { + const url = this.markPendingUrlValue.replace('__PART_ID__', partId) + const data = await this.fetchWithErrorHandling(url, { method: 'POST' }) + + if (data.success) { + this.updateProgressDisplay(data) + this.markRowAsPending(partId) + } else { + this.showErrorMessage(data.error || 'Failed to mark part as pending') + } + } catch (error) { + console.error('Error marking part as pending:', error) + this.showErrorMessage(error.message || 'Failed to mark part as pending') + } + } + + updateProgressDisplay(data) { + if (this.hasProgressBarTarget) { + this.progressBarTarget.style.width = `${data.progress}%` + this.progressBarTarget.setAttribute('aria-valuenow', data.progress) + } + + if (this.hasProgressTextTarget) { + this.progressTextTarget.textContent = `${data.completed_count} / ${data.total_count} completed` + } + } + + markRowAsCompleted(partId) { + // Save scroll position and refresh page to show updated state + sessionStorage.setItem('bulkImportScrollPosition', window.scrollY.toString()) + window.location.reload() + } + + markRowAsSkipped(partId) { + // Save scroll position and refresh page to show updated state + sessionStorage.setItem('bulkImportScrollPosition', window.scrollY.toString()) + window.location.reload() + } + + markRowAsPending(partId) { + // Save scroll position and refresh page to show updated state + sessionStorage.setItem('bulkImportScrollPosition', window.scrollY.toString()) + window.location.reload() + } + + showJobCompletedMessage() { + const alert = document.createElement('div') + alert.className = 'alert alert-success alert-dismissible fade show' + alert.innerHTML = ` + + Job completed! All parts have been processed. + + ` + + const container = document.querySelector('.card-body') + container.insertBefore(alert, container.firstChild) + } + + async researchPart(event) { + event.preventDefault() + event.stopPropagation() + + const partId = event.currentTarget.dataset.partId + const spinner = event.currentTarget.querySelector(`[data-research-spinner="${partId}"]`) + const button = event.currentTarget + + // Show loading state + if (spinner) { + spinner.style.display = 'inline-block' + } + button.disabled = true + + try { + const url = this.researchUrlValue.replace('__PART_ID__', partId) + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), 30000) // 30 second timeout + + const response = await fetch(url, { + method: 'POST', + headers: this.getHeaders(), + signal: controller.signal + }) + + clearTimeout(timeoutId) + + if (!response.ok) { + const errorText = await response.text() + throw new Error(`Server error (${response.status}): ${errorText}`) + } + + const data = await response.json() + + if (data.success) { + this.showSuccessMessage(`Research completed for part. Found ${data.results_count} results.`) + // Save scroll position and reload to show updated results + sessionStorage.setItem('bulkImportScrollPosition', window.scrollY.toString()) + window.location.reload() + } else { + this.showErrorMessage(data.error || 'Research failed') + } + } catch (error) { + console.error('Error researching part:', error) + + if (error.name === 'AbortError') { + this.showErrorMessage('Research timed out. Please try again.') + } else if (error.message.includes('Failed to fetch')) { + this.showErrorMessage('Network error. Please check your connection and try again.') + } else { + this.showErrorMessage(error.message || 'Research failed due to an unexpected error') + } + } finally { + // Hide loading state + if (spinner) { + spinner.style.display = 'none' + } + button.disabled = false + } + } + + async researchAllParts(event) { + event.preventDefault() + event.stopPropagation() + + const spinner = document.getElementById('research-all-spinner') + const button = event.currentTarget + + // Show loading state + if (spinner) { + spinner.style.display = 'inline-block' + } + button.disabled = true + + try { + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), 120000) // 2 minute timeout for bulk operations + + const response = await fetch(this.researchAllUrlValue, { + method: 'POST', + headers: this.getHeaders(), + signal: controller.signal + }) + + clearTimeout(timeoutId) + + if (!response.ok) { + const errorText = await response.text() + throw new Error(`Server error (${response.status}): ${errorText}`) + } + + const data = await response.json() + + if (data.success) { + this.showSuccessMessage(`Research completed for ${data.researched_count} parts.`) + // Save scroll position and reload to show updated results + sessionStorage.setItem('bulkImportScrollPosition', window.scrollY.toString()) + window.location.reload() + } else { + this.showErrorMessage(data.error || 'Bulk research failed') + } + } catch (error) { + console.error('Error researching all parts:', error) + + if (error.name === 'AbortError') { + this.showErrorMessage('Bulk research timed out. This may happen with large batches. Please try again or process smaller batches.') + } else if (error.message.includes('Failed to fetch')) { + this.showErrorMessage('Network error. Please check your connection and try again.') + } else { + this.showErrorMessage(error.message || 'Bulk research failed due to an unexpected error') + } + } finally { + // Hide loading state + if (spinner) { + spinner.style.display = 'none' + } + button.disabled = false + } + } + + showSuccessMessage(message) { + this.showToast('success', message) + } + + showErrorMessage(message) { + this.showToast('error', message) + } + + showToast(type, message) { + // Create a simple alert that doesn't disrupt layout + const alertId = 'alert-' + Date.now() + const iconClass = type === 'success' ? 'fa-check-circle' : 'fa-exclamation-triangle' + const alertClass = type === 'success' ? 'alert-success' : 'alert-danger' + + const alertHTML = ` +
+ + ${message} + +
+ ` + + // Add alert to body + document.body.insertAdjacentHTML('beforeend', alertHTML) + + // Auto-remove after 5 seconds + setTimeout(() => { + const alertElement = document.getElementById(alertId) + if (alertElement) { + alertElement.remove() + } + }, 5000) + } +} \ No newline at end of file diff --git a/assets/controllers/bulk_job_manage_controller.js b/assets/controllers/bulk_job_manage_controller.js new file mode 100644 index 00000000..c26e37c6 --- /dev/null +++ b/assets/controllers/bulk_job_manage_controller.js @@ -0,0 +1,92 @@ +import { Controller } from "@hotwired/stimulus" +import { generateCsrfHeaders } from "./csrf_protection_controller" + +export default class extends Controller { + static values = { + deleteUrl: String, + stopUrl: String, + deleteConfirmMessage: String, + stopConfirmMessage: String + } + + connect() { + // Controller initialized + } + getHeaders() { + const headers = { + 'X-Requested-With': 'XMLHttpRequest' + } + + // Add CSRF headers if available + const form = document.querySelector('form') + if (form) { + const csrfHeaders = generateCsrfHeaders(form) + Object.assign(headers, csrfHeaders) + } + + return headers + } + async deleteJob(event) { + const jobId = event.currentTarget.dataset.jobId + const confirmMessage = this.deleteConfirmMessageValue || 'Are you sure you want to delete this job?' + + if (confirm(confirmMessage)) { + try { + const deleteUrl = this.deleteUrlValue.replace('__JOB_ID__', jobId) + + const response = await fetch(deleteUrl, { + method: 'DELETE', + headers: this.getHeaders() + }) + + if (!response.ok) { + const errorText = await response.text() + throw new Error(`HTTP ${response.status}: ${errorText}`) + } + + const data = await response.json() + + if (data.success) { + location.reload() + } else { + alert('Error deleting job: ' + (data.error || 'Unknown error')) + } + } catch (error) { + console.error('Error deleting job:', error) + alert('Error deleting job: ' + error.message) + } + } + } + + async stopJob(event) { + const jobId = event.currentTarget.dataset.jobId + const confirmMessage = this.stopConfirmMessageValue || 'Are you sure you want to stop this job?' + + if (confirm(confirmMessage)) { + try { + const stopUrl = this.stopUrlValue.replace('__JOB_ID__', jobId) + + const response = await fetch(stopUrl, { + method: 'POST', + headers: this.getHeaders() + }) + + if (!response.ok) { + const errorText = await response.text() + throw new Error(`HTTP ${response.status}: ${errorText}`) + } + + const data = await response.json() + + if (data.success) { + location.reload() + } else { + alert('Error stopping job: ' + (data.error || 'Unknown error')) + } + } catch (error) { + console.error('Error stopping job:', error) + alert('Error stopping job: ' + error.message) + } + } + } +} \ No newline at end of file diff --git a/assets/controllers/field_mapping_controller.js b/assets/controllers/field_mapping_controller.js new file mode 100644 index 00000000..738035c2 --- /dev/null +++ b/assets/controllers/field_mapping_controller.js @@ -0,0 +1,138 @@ +import { Controller } from "@hotwired/stimulus" + +export default class extends Controller { + static targets = ["tbody", "addButton", "submitButton"] + static values = { + mappingIndex: Number, + maxMappings: Number, + prototype: String, + maxMappingsReachedMessage: String + } + + connect() { + this.updateAddButtonState() + this.updateFieldOptions() + this.attachEventListeners() + } + + attachEventListeners() { + // Add event listeners to existing field selects + const fieldSelects = this.tbodyTarget.querySelectorAll('select[name*="[field]"]') + fieldSelects.forEach(select => { + select.addEventListener('change', this.updateFieldOptions.bind(this)) + }) + + // Add click listener to add button + if (this.hasAddButtonTarget) { + this.addButtonTarget.addEventListener('click', this.addMapping.bind(this)) + } + + // Form submit handler + const form = this.element.querySelector('form') + if (form && this.hasSubmitButtonTarget) { + form.addEventListener('submit', this.handleFormSubmit.bind(this)) + } + } + + addMapping() { + const currentMappings = this.tbodyTarget.querySelectorAll('.mapping-row').length + + if (currentMappings >= this.maxMappingsValue) { + alert(this.maxMappingsReachedMessageValue) + return + } + + const newRowHtml = this.prototypeValue.replace(/__name__/g, this.mappingIndexValue) + const tempDiv = document.createElement('div') + tempDiv.innerHTML = newRowHtml + + const fieldWidget = tempDiv.querySelector('select[name*="[field]"]') || tempDiv.children[0] + const providerWidget = tempDiv.querySelector('select[name*="[providers]"]') || tempDiv.children[1] + const priorityWidget = tempDiv.querySelector('input[name*="[priority]"]') || tempDiv.children[2] + + const newRow = document.createElement('tr') + newRow.className = 'mapping-row' + newRow.innerHTML = ` + ${fieldWidget ? fieldWidget.outerHTML : ''} + ${providerWidget ? providerWidget.outerHTML : ''} + ${priorityWidget ? priorityWidget.outerHTML : ''} + + + + ` + + this.tbodyTarget.appendChild(newRow) + this.mappingIndexValue++ + + const newFieldSelect = newRow.querySelector('select[name*="[field]"]') + if (newFieldSelect) { + newFieldSelect.value = '' + newFieldSelect.addEventListener('change', this.updateFieldOptions.bind(this)) + } + + this.updateFieldOptions() + this.updateAddButtonState() + } + + removeMapping(event) { + const row = event.target.closest('tr') + row.remove() + this.updateFieldOptions() + this.updateAddButtonState() + } + + updateFieldOptions() { + const fieldSelects = this.tbodyTarget.querySelectorAll('select[name*="[field]"]') + + const selectedFields = Array.from(fieldSelects) + .map(select => select.value) + .filter(value => value && value !== '') + + fieldSelects.forEach(select => { + Array.from(select.options).forEach(option => { + const isCurrentValue = option.value === select.value + const isEmptyOption = !option.value || option.value === '' + const isAlreadySelected = selectedFields.includes(option.value) + + if (!isEmptyOption && isAlreadySelected && !isCurrentValue) { + option.disabled = true + option.style.display = 'none' + } else { + option.disabled = false + option.style.display = '' + } + }) + }) + } + + updateAddButtonState() { + const currentMappings = this.tbodyTarget.querySelectorAll('.mapping-row').length + + if (this.hasAddButtonTarget) { + if (currentMappings >= this.maxMappingsValue) { + this.addButtonTarget.disabled = true + this.addButtonTarget.title = this.maxMappingsReachedMessageValue + } else { + this.addButtonTarget.disabled = false + this.addButtonTarget.title = '' + } + } + } + + handleFormSubmit(event) { + if (this.hasSubmitButtonTarget) { + this.submitButtonTarget.disabled = true + + // Disable the entire form to prevent changes during processing + const form = event.target + const formElements = form.querySelectorAll('input, select, textarea, button') + formElements.forEach(element => { + if (element !== this.submitButtonTarget) { + element.disabled = true + } + }) + } + } +} \ No newline at end of file diff --git a/src/Controller/BulkInfoProviderImportController.php b/src/Controller/BulkInfoProviderImportController.php index 880bfe5f..38771e40 100644 --- a/src/Controller/BulkInfoProviderImportController.php +++ b/src/Controller/BulkInfoProviderImportController.php @@ -28,16 +28,14 @@ use App\Entity\BulkImportJobStatus; use App\Entity\Parts\Part; use App\Entity\Parts\Supplier; use App\Form\InfoProviderSystem\GlobalFieldMappingType; -use App\Services\InfoProviderSystem\PartInfoRetriever; -use App\Services\InfoProviderSystem\ExistingPartFinder; -use App\Services\InfoProviderSystem\ProviderRegistry; -use App\Services\InfoProviderSystem\Providers\LCSCProvider; +use App\Services\InfoProviderSystem\BulkInfoProviderService; +use App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO; use Doctrine\ORM\EntityManagerInterface; use Psr\Log\LoggerInterface; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; -use Symfony\Component\HttpClient\Exception\ClientException; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; +use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\Routing\Attribute\Route; use App\Entity\UserSystem\User; @@ -45,20 +43,67 @@ use App\Entity\UserSystem\User; class BulkInfoProviderImportController extends AbstractController { public function __construct( - private readonly PartInfoRetriever $infoRetriever, - private readonly LCSCProvider $LCSCProvider, - private readonly ExistingPartFinder $existingPartFinder, - private readonly EntityManagerInterface $entityManager + private readonly BulkInfoProviderService $bulkService, + private readonly EntityManagerInterface $entityManager, + private readonly LoggerInterface $logger ) { } + private function createErrorResponse(string $message, int $statusCode = 400, array $context = []): JsonResponse + { + $this->logger->warning('Bulk import operation failed', array_merge([ + 'error' => $message, + 'user' => $this->getUser()?->getUserIdentifier(), + ], $context)); + + return $this->json([ + 'success' => false, + 'error' => $message + ], $statusCode); + } + + private function validateJobAccess(int $jobId): ?BulkInfoProviderImportJob + { + $job = $this->entityManager->getRepository(BulkInfoProviderImportJob::class)->find($jobId); + + if (!$job) { + return null; + } + + if ($job->getCreatedBy() !== $this->getUser()) { + return null; + } + + return $job; + } + + private function updatePartSearchResults(BulkInfoProviderImportJob $job, int $partId, ?array $newResults): void + { + if ($newResults === null) { + return; + } + + // Only deserialize and update if we have new results + $allResults = $job->deserializeSearchResults($this->entityManager); + + // Find and update the results for this specific part + foreach ($allResults as $index => $partResult) { + if ($partResult['part']->getId() === $partId) { + $allResults[$index] = $newResults; + break; + } + } + + // Save updated results back to job + $job->setSearchResults($job->serializeSearchResults($allResults)); + } + #[Route('/step1', name: 'bulk_info_provider_step1')] - public function step1(Request $request, LoggerInterface $exceptionLogger): Response + public function step1(Request $request): Response { $this->denyAccessUnlessGranted('@info_providers.create_parts'); - // Increase execution time for bulk operations - set_time_limit(600); // 10 minutes for large batches + set_time_limit(600); $ids = $request->query->get('ids'); if (!$ids) { @@ -66,7 +111,6 @@ class BulkInfoProviderImportController extends AbstractController return $this->redirectToRoute('homepage'); } - // Get the selected parts $partIds = explode(',', $ids); $partRepository = $this->entityManager->getRepository(Part::class); $parts = $partRepository->getElementsFromIDArray($partIds); @@ -76,7 +120,6 @@ class BulkInfoProviderImportController extends AbstractController return $this->redirectToRoute('homepage'); } - // Warn about large batches if (count($parts) > 50) { $this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.'); } @@ -114,23 +157,17 @@ class BulkInfoProviderImportController extends AbstractController $fieldMappings = $formData['field_mappings']; $prefetchDetails = $formData['prefetch_details'] ?? false; - // Debug logging - $exceptionLogger->info('Form data received', [ - 'prefetch_details' => $prefetchDetails, - 'prefetch_details_type' => gettype($prefetchDetails) - ]); + $user = $this->getUser(); + if (!$user instanceof User) { + throw new \RuntimeException('User must be authenticated and of type User'); + } // Create and save the job $job = new BulkInfoProviderImportJob(); $job->setFieldMappings($fieldMappings); $job->setPrefetchDetails($prefetchDetails); - $user = $this->getUser(); - if (!$user instanceof User) { - throw new \RuntimeException('User must be authenticated and of type User'); - } $job->setCreatedBy($user); - // Create job parts for each part foreach ($parts as $part) { $jobPart = new BulkInfoProviderImportJobPart($job, $part); $job->addJobPart($jobPart); @@ -139,200 +176,40 @@ class BulkInfoProviderImportController extends AbstractController $this->entityManager->persist($job); $this->entityManager->flush(); - $searchResults = []; - $hasAnyResults = false; - try { - // Optimize: Use batch async requests for LCSC provider - $lcscKeywords = []; - $keywordToPartField = []; + $searchRequest = new BulkSearchRequestDTO( + fieldMappings: $fieldMappings, + prefetchDetails: $prefetchDetails, + partIds: $partIds + ); - // First, collect all LCSC keywords for batch processing - foreach ($parts as $part) { - foreach ($fieldMappings as $mapping) { - $field = $mapping['field']; - $providers = $mapping['providers'] ?? []; - - if (in_array('lcsc', $providers, true)) { - $keyword = $this->getKeywordFromField($part, $field); - if ($keyword) { - $lcscKeywords[] = $keyword; - $keywordToPartField[$keyword] = [ - 'part' => $part, - 'field' => $field - ]; - } - } - } - } - - // Batch search LCSC keywords asynchronously - $lcscBatchResults = []; - if (!empty($lcscKeywords)) { - try { - // Try to get LCSC provider and use batch method if available - $lcscBatchResults = $this->searchLcscBatch($lcscKeywords); - } catch (\Exception $e) { - $exceptionLogger->warning('LCSC batch search failed, falling back to individual requests', [ - 'error' => $e->getMessage() - ]); - } - } - - // Now process each part - foreach ($parts as $part) { - $partResult = [ - 'part' => $part, - 'search_results' => [], - 'errors' => [] - ]; - - // Collect all DTOs using priority-based search - $allDtos = []; - $dtoMetadata = []; // Store source field info separately - - // Group mappings by priority (lower number = higher priority) - $mappingsByPriority = []; - foreach ($fieldMappings as $mapping) { - $priority = $mapping['priority'] ?? 1; - $mappingsByPriority[$priority][] = $mapping; - } - ksort($mappingsByPriority); // Sort by priority (1, 2, 3...) - - // Try each priority level until we find results - foreach ($mappingsByPriority as $priority => $mappings) { - $priorityResults = []; - - // For same priority, search all and combine results - foreach ($mappings as $mapping) { - $field = $mapping['field']; - $providers = $mapping['providers'] ?? []; - - if (empty($providers)) { - continue; - } - - $keyword = $this->getKeywordFromField($part, $field); - - if ($keyword) { - try { - // Use batch results for LCSC if available - if (in_array('lcsc', $providers, true) && isset($lcscBatchResults[$keyword])) { - $dtos = $lcscBatchResults[$keyword]; - } else { - // Fall back to regular search for non-LCSC providers - $dtos = $this->infoRetriever->searchByKeyword( - keyword: $keyword, - providers: $providers - ); - } - - // Store field info for each DTO separately - foreach ($dtos as $dto) { - $dtoKey = $dto->provider_key . '|' . $dto->provider_id; - $dtoMetadata[$dtoKey] = [ - 'source_field' => $field, - 'source_keyword' => $keyword, - 'priority' => $priority - ]; - } - - $priorityResults = array_merge($priorityResults, $dtos); - } catch (ClientException $e) { - $partResult['errors'][] = "Error searching with {$field} (priority {$priority}): " . $e->getMessage(); - $exceptionLogger->error('Error during bulk info provider search for part ' . $part->getId() . " field {$field}: " . $e->getMessage(), ['exception' => $e]); - } - } - } - - // If we found results at this priority level, use them and stop - if (!empty($priorityResults)) { - $allDtos = $priorityResults; - break; - } - } - - // Remove duplicates based on provider_key + provider_id - $uniqueDtos = []; - $seenKeys = []; - foreach ($allDtos as $dto) { - if ($dto === null || !isset($dto->provider_key, $dto->provider_id)) { - continue; - } - $key = "{$dto->provider_key}|{$dto->provider_id}"; - if (!in_array($key, $seenKeys, true)) { - $seenKeys[] = $key; - $uniqueDtos[] = $dto; - } - } - - // Convert DTOs to result format with metadata - $partResult['search_results'] = array_map( - function ($dto) use ($dtoMetadata) { - $dtoKey = $dto->provider_key . '|' . $dto->provider_id; - $metadata = $dtoMetadata[$dtoKey] ?? []; - return [ - 'dto' => $dto, - 'localPart' => $this->existingPartFinder->findFirstExisting($dto), - 'source_field' => $metadata['source_field'] ?? null, - 'source_keyword' => $metadata['source_keyword'] ?? null - ]; - }, - $uniqueDtos - ); - - if (!empty($partResult['search_results'])) { - $hasAnyResults = true; - } - - $searchResults[] = $partResult; - } - - // Check if search was successful - if (!$hasAnyResults) { - $exceptionLogger->warning('Bulk import search returned no results for any parts', [ - 'job_id' => $job->getId(), - 'parts_count' => count($parts) - ]); - - // Delete the job since it has no useful results - $this->entityManager->remove($job); - $this->entityManager->flush(); - - $this->addFlash('error', 'No search results found for any of the selected parts. Please check your field mappings and provider selections.'); - return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]); - } + $searchResults = $this->bulkService->performBulkSearch($searchRequest); // Save search results to job - $job->setSearchResults($this->serializeSearchResults($searchResults)); + $job->setSearchResults($job->serializeSearchResults($searchResults)); $job->markAsInProgress(); $this->entityManager->flush(); + // Prefetch details if requested + if ($prefetchDetails) { + $this->bulkService->prefetchDetailsForResults($searchResults); + } + + return $this->redirectToRoute('bulk_info_provider_step2', ['jobId' => $job->getId()]); + } catch (\Exception $e) { - $exceptionLogger->error('Critical error during bulk import search', [ + $this->logger->error('Critical error during bulk import search', [ 'job_id' => $job->getId(), 'error' => $e->getMessage(), 'exception' => $e ]); - // Delete the job on critical failure $this->entityManager->remove($job); $this->entityManager->flush(); $this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage()); return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]); } - - // Prefetch details if requested - if ($prefetchDetails) { - $exceptionLogger->info('Prefetch details requested, starting prefetch for ' . count($searchResults) . ' parts'); - $this->prefetchDetailsForResults($searchResults, $exceptionLogger); - } else { - $exceptionLogger->info('Prefetch details not requested, skipping prefetch'); - } - - // Redirect to step 2 with the job - return $this->redirectToRoute('bulk_info_provider_step2', ['jobId' => $job->getId()]); } // Get existing in-progress jobs for current user @@ -433,72 +310,6 @@ class BulkInfoProviderImportController extends AbstractController return $this->json(['success' => true]); } - private function getKeywordFromField(Part $part, string $field): ?string - { - return match ($field) { - 'mpn' => $part->getManufacturerProductNumber(), - 'name' => $part->getName(), - default => $this->getSupplierPartNumber($part, $field) - }; - } - - private function getSupplierPartNumber(Part $part, string $field): ?string - { - // Check if this is a supplier SPN field - if (!str_ends_with($field, '_spn')) { - return null; - } - - // Extract supplier key (remove _spn suffix) - $supplierKey = substr($field, 0, -4); - - // Get all suppliers to find matching one - $suppliers = $this->entityManager->getRepository(Supplier::class)->findAll(); - - foreach ($suppliers as $supplier) { - $normalizedName = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName())); - if ($normalizedName === $supplierKey) { - // Find order detail for this supplier - $orderDetail = $part->getOrderdetails()->filter( - fn($od) => $od->getSupplier()?->getId() === $supplier->getId() - )->first(); - - return $orderDetail ? $orderDetail->getSupplierpartnr() : null; - } - } - - return null; - } - - /** - * Prefetch details for all search results to populate cache - */ - private function prefetchDetailsForResults(array $searchResults, LoggerInterface $logger): void - { - $prefetchCount = 0; - - foreach ($searchResults as $partResult) { - foreach ($partResult['search_results'] as $result) { - $dto = $result['dto']; - - try { - // This call will cache the details for later use - $this->infoRetriever->getDetails($dto->provider_key, $dto->provider_id); - $prefetchCount++; - } catch (\Exception $e) { - $logger->warning('Failed to prefetch details for provider part', [ - 'provider_key' => $dto->provider_key, - 'provider_id' => $dto->provider_id, - 'error' => $e->getMessage() - ]); - } - } - } - - if ($prefetchCount > 0) { - $this->addFlash('success', "Prefetched details for {$prefetchCount} search results"); - } - } #[Route('/step2/{jobId}', name: 'bulk_info_provider_step2')] public function step2(int $jobId): Response @@ -518,7 +329,7 @@ class BulkInfoProviderImportController extends AbstractController // Get the parts and deserialize search results $parts = $job->getJobParts()->map(fn($jobPart) => $jobPart->getPart())->toArray(); - $searchResults = $this->deserializeSearchResults($job->getSearchResults(), $parts); + $searchResults = $job->deserializeSearchResults($this->entityManager); return $this->render('info_providers/bulk_import/step2.html.twig', [ 'job' => $job, @@ -527,103 +338,6 @@ class BulkInfoProviderImportController extends AbstractController ]); } - private function serializeSearchResults(array $searchResults): array - { - $serialized = []; - - foreach ($searchResults as $partResult) { - $partData = [ - 'part_id' => $partResult['part']->getId(), - 'search_results' => [], - 'errors' => $partResult['errors'] - ]; - - foreach ($partResult['search_results'] as $result) { - $dto = $result['dto']; - $partData['search_results'][] = [ - 'dto' => [ - 'provider_key' => $dto->provider_key, - 'provider_id' => $dto->provider_id, - 'name' => $dto->name, - 'description' => $dto->description, - 'manufacturer' => $dto->manufacturer, - 'mpn' => $dto->mpn, - 'provider_url' => $dto->provider_url, - 'preview_image_url' => $dto->preview_image_url, - '_source_field' => $result['source_field'] ?? null, - '_source_keyword' => $result['source_keyword'] ?? null, - ], - 'localPart' => $result['localPart'] ? $result['localPart']->getId() : null - ]; - } - - $serialized[] = $partData; - } - - return $serialized; - } - - private function deserializeSearchResults(array $serializedResults, array $parts): array - { - $partsById = []; - foreach ($parts as $part) { - $partsById[$part->getId()] = $part; - } - - $searchResults = []; - - foreach ($serializedResults as $partData) { - $part = $partsById[$partData['part_id']] ?? null; - if (!$part) { - continue; - } - - $partResult = [ - 'part' => $part, - 'search_results' => [], - 'errors' => $partData['errors'] - ]; - - foreach ($partData['search_results'] as $resultData) { - $dtoData = $resultData['dto']; - - $dto = new \App\Services\InfoProviderSystem\DTOs\SearchResultDTO( - provider_key: $dtoData['provider_key'], - provider_id: $dtoData['provider_id'], - name: $dtoData['name'], - description: $dtoData['description'], - manufacturer: $dtoData['manufacturer'], - mpn: $dtoData['mpn'], - provider_url: $dtoData['provider_url'], - preview_image_url: $dtoData['preview_image_url'] - ); - - $localPart = null; - if ($resultData['localPart']) { - $localPart = $this->entityManager->getRepository(Part::class)->find($resultData['localPart']); - } - - $partResult['search_results'][] = [ - 'dto' => $dto, - 'localPart' => $localPart, - 'source_field' => $dtoData['_source_field'] ?? null, - 'source_keyword' => $dtoData['_source_keyword'] ?? null - ]; - } - - $searchResults[] = $partResult; - } - - return $searchResults; - } - - /** - * Perform batch LCSC search using async HTTP requests - */ - private function searchLcscBatch(array $keywords): array - { - return $this->LCSCProvider->searchByKeywordsBatch($keywords); - } #[Route('/job/{jobId}/part/{partId}/mark-completed', name: 'bulk_info_provider_mark_completed', methods: ['POST'])] public function markPartCompleted(int $jobId, int $partId): Response @@ -702,4 +416,155 @@ class BulkInfoProviderImportController extends AbstractController 'job_completed' => $job->isCompleted() ]); } + + #[Route('/job/{jobId}/part/{partId}/research', name: 'bulk_info_provider_research_part', methods: ['POST'])] + public function researchPart(int $jobId, int $partId): JsonResponse + { + $job = $this->validateJobAccess($jobId); + if (!$job) { + return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]); + } + + $part = $this->entityManager->getRepository(Part::class)->find($partId); + if (!$part) { + return $this->createErrorResponse('Part not found', 404, ['part_id' => $partId]); + } + + // Only refresh if the entity might be stale (optional optimization) + if ($this->entityManager->getUnitOfWork()->isScheduledForUpdate($part)) { + $this->entityManager->refresh($part); + } + + try { + // Use the job's field mappings to perform the search + $fieldMappings = $job->getFieldMappings(); + $prefetchDetails = $job->isPrefetchDetails(); + + $searchRequest = new BulkSearchRequestDTO( + fieldMappings: $fieldMappings, + prefetchDetails: $prefetchDetails, + partIds: [$partId] + ); + + try { + $searchResults = $this->bulkService->performBulkSearch($searchRequest); + } catch (\Exception $searchException) { + // Handle "no search results found" as a normal case, not an error + if (str_contains($searchException->getMessage(), 'No search results found')) { + $searchResults = []; + } else { + throw $searchException; + } + } + + // Update the job's search results for this specific part efficiently + $this->updatePartSearchResults($job, $partId, $searchResults[0] ?? null); + + // Prefetch details if requested + if ($prefetchDetails && !empty($searchResults)) { + $this->bulkService->prefetchDetailsForResults($searchResults); + } + + $this->entityManager->flush(); + + // Return the new results for this part + $newResults = $searchResults[0] ?? null; + + return $this->json([ + 'success' => true, + 'part_id' => $partId, + 'results_count' => $newResults ? count($newResults['search_results']) : 0, + 'errors_count' => $newResults ? count($newResults['errors']) : 0, + 'message' => 'Part research completed successfully' + ]); + + } catch (\Exception $e) { + return $this->createErrorResponse( + 'Research failed: ' . $e->getMessage(), + 500, + [ + 'job_id' => $jobId, + 'part_id' => $partId, + 'exception' => $e->getMessage() + ] + ); + } + } + + #[Route('/job/{jobId}/research-all', name: 'bulk_info_provider_research_all', methods: ['POST'])] + public function researchAllParts(int $jobId): JsonResponse + { + $job = $this->validateJobAccess($jobId); + if (!$job) { + return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]); + } + + // Get all part IDs that are not completed or skipped + $partIds = []; + foreach ($job->getJobParts() as $jobPart) { + if (!$jobPart->isCompleted() && !$jobPart->isSkipped()) { + $partIds[] = $jobPart->getPart()->getId(); + } + } + + if (empty($partIds)) { + return $this->json([ + 'success' => true, + 'message' => 'No parts to research', + 'researched_count' => 0 + ]); + } + + try { + $fieldMappings = $job->getFieldMappings(); + $prefetchDetails = $job->isPrefetchDetails(); + + // Process in batches to reduce memory usage for large operations + $batchSize = 20; // Configurable batch size for memory management + $allResults = []; + $batches = array_chunk($partIds, $batchSize); + + foreach ($batches as $batch) { + $searchRequest = new BulkSearchRequestDTO( + fieldMappings: $fieldMappings, + prefetchDetails: $prefetchDetails, + partIds: $batch + ); + + $batchResults = $this->bulkService->performBulkSearch($searchRequest); + $allResults = array_merge($allResults, $batchResults); + + // Clear entity manager periodically to prevent memory issues + $this->entityManager->clear(); + $job = $this->entityManager->find(BulkInfoProviderImportJob::class, $job->getId()); + } + + // Update the job's search results + $job->setSearchResults($job->serializeSearchResults($allResults)); + + // Prefetch details if requested + if ($prefetchDetails) { + $this->bulkService->prefetchDetailsForResults($allResults); + } + + $this->entityManager->flush(); + + return $this->json([ + 'success' => true, + 'researched_count' => count($partIds), + 'message' => sprintf('Successfully researched %d parts', count($partIds)) + ]); + + } catch (\Exception $e) { + return $this->createErrorResponse( + 'Bulk research failed: ' . $e->getMessage(), + 500, + [ + 'job_id' => $jobId, + 'part_ids' => $partIds, + 'exception' => $e->getMessage() + ] + ); + } + } } diff --git a/src/Entity/BulkInfoProviderImportJob.php b/src/Entity/BulkInfoProviderImportJob.php index 2a602030..24fc8692 100644 --- a/src/Entity/BulkInfoProviderImportJob.php +++ b/src/Entity/BulkInfoProviderImportJob.php @@ -25,10 +25,12 @@ namespace App\Entity; use App\Entity\Base\AbstractDBElement; use App\Entity\Parts\Part; use App\Entity\UserSystem\User; +use App\Services\InfoProviderSystem\DTOs\SearchResultDTO; use Doctrine\Common\Collections\ArrayCollection; use Doctrine\Common\Collections\Collection; use Doctrine\DBAL\Types\Types; use Doctrine\ORM\Mapping as ORM; +use Doctrine\ORM\EntityManagerInterface; enum BulkImportJobStatus: string { @@ -403,4 +405,99 @@ class BulkInfoProviderImportJob extends AbstractDBElement $completed = $this->getCompletedPartsCount() + $this->getSkippedPartsCount(); return $completed >= $total; } + + public function serializeSearchResults(array $searchResults): array + { + $serialized = []; + + foreach ($searchResults as $partResult) { + $partData = [ + 'part_id' => $partResult['part']->getId(), + 'search_results' => [], + 'errors' => $partResult['errors'] ?? [] + ]; + + foreach ($partResult['search_results'] as $result) { + $dto = $result['dto']; + $partData['search_results'][] = [ + 'dto' => [ + 'provider_key' => $dto->provider_key, + 'provider_id' => $dto->provider_id, + 'name' => $dto->name, + 'description' => $dto->description, + 'manufacturer' => $dto->manufacturer, + 'mpn' => $dto->mpn, + 'provider_url' => $dto->provider_url, + 'preview_image_url' => $dto->preview_image_url, + '_source_field' => $result['source_field'] ?? null, + '_source_keyword' => $result['source_keyword'] ?? null, + ], + 'localPart' => $result['localPart'] ? $result['localPart']->getId() : null + ]; + } + + $serialized[] = $partData; + } + + return $serialized; + } + + public function deserializeSearchResults(?EntityManagerInterface $entityManager = null): array + { + if (empty($this->searchResults)) { + return []; + } + + $parts = $this->jobParts->map(fn($jobPart) => $jobPart->getPart())->toArray(); + $partsById = []; + foreach ($parts as $part) { + $partsById[$part->getId()] = $part; + } + + $searchResults = []; + + foreach ($this->searchResults as $partData) { + $part = $partsById[$partData['part_id']] ?? null; + if (!$part) { + continue; + } + + $partResult = [ + 'part' => $part, + 'search_results' => [], + 'errors' => $partData['errors'] ?? [] + ]; + + foreach ($partData['search_results'] as $resultData) { + $dtoData = $resultData['dto']; + + $dto = new SearchResultDTO( + provider_key: $dtoData['provider_key'], + provider_id: $dtoData['provider_id'], + name: $dtoData['name'], + description: $dtoData['description'], + manufacturer: $dtoData['manufacturer'], + mpn: $dtoData['mpn'], + provider_url: $dtoData['provider_url'], + preview_image_url: $dtoData['preview_image_url'] + ); + + $localPart = null; + if ($resultData['localPart'] && $entityManager) { + $localPart = $entityManager->getRepository(Part::class)->find($resultData['localPart']); + } + + $partResult['search_results'][] = [ + 'dto' => $dto, + 'localPart' => $localPart, + 'source_field' => $dtoData['_source_field'] ?? null, + 'source_keyword' => $dtoData['_source_keyword'] ?? null + ]; + } + + $searchResults[] = $partResult; + } + + return $searchResults; + } } \ No newline at end of file diff --git a/src/Form/InfoProviderSystem/GlobalFieldMappingType.php b/src/Form/InfoProviderSystem/GlobalFieldMappingType.php index 1f2af5b1..ea70284f 100644 --- a/src/Form/InfoProviderSystem/GlobalFieldMappingType.php +++ b/src/Form/InfoProviderSystem/GlobalFieldMappingType.php @@ -54,7 +54,7 @@ class GlobalFieldMappingType extends AbstractType ]); $builder->add('submit', SubmitType::class, [ - 'label' => 'info_providers.bulk_search.submit' + 'label' => 'info_providers.bulk_import.search.submit' ]); } diff --git a/src/Form/InfoProviderSystem/ProviderSelectType.php b/src/Form/InfoProviderSystem/ProviderSelectType.php index a9373390..ddd70bc6 100644 --- a/src/Form/InfoProviderSystem/ProviderSelectType.php +++ b/src/Form/InfoProviderSystem/ProviderSelectType.php @@ -24,9 +24,7 @@ declare(strict_types=1); namespace App\Form\InfoProviderSystem; use App\Services\InfoProviderSystem\ProviderRegistry; -use App\Services\InfoProviderSystem\Providers\InfoProviderInterface; use Symfony\Component\Form\AbstractType; -use Symfony\Component\Form\ChoiceList\ChoiceList; use Symfony\Component\Form\Extension\Core\Type\ChoiceType; use Symfony\Component\OptionsResolver\OptionsResolver; @@ -44,11 +42,16 @@ class ProviderSelectType extends AbstractType public function configureOptions(OptionsResolver $resolver): void { + $providers = $this->providerRegistry->getActiveProviders(); + + // Create a simple array of provider keys => labels + $choices = []; + foreach ($providers as $provider) { + $choices[$provider->getProviderInfo()['name']] = $provider->getProviderKey(); + } + $resolver->setDefaults([ - 'choices' => $this->providerRegistry->getActiveProviders(), - 'choice_label' => ChoiceList::label($this, static fn (?InfoProviderInterface $choice) => $choice?->getProviderInfo()['name']), - 'choice_value' => ChoiceList::value($this, static fn(?InfoProviderInterface $choice) => $choice?->getProviderKey()), - + 'choices' => $choices, 'multiple' => true, ]); } diff --git a/src/Services/InfoProviderSystem/BulkInfoProviderService.php b/src/Services/InfoProviderSystem/BulkInfoProviderService.php new file mode 100644 index 00000000..b78ca450 --- /dev/null +++ b/src/Services/InfoProviderSystem/BulkInfoProviderService.php @@ -0,0 +1,297 @@ +partIds); + + $partRepository = $this->entityManager->getRepository(Part::class); + $parts = $partRepository->getElementsFromIDArray($partIds); + + if (empty($parts)) { + throw new \InvalidArgumentException('No valid parts found for bulk import'); + } + + $searchResults = []; + $hasAnyResults = false; + + // Group providers by batch capability + $batchProviders = []; + $regularProviders = []; + + foreach ($request->fieldMappings as $mapping) { + $providers = $mapping['providers'] ?? []; + foreach ($providers as $providerKey) { + if (!is_string($providerKey)) { + $this->logger->error('Invalid provider key type', [ + 'providerKey' => $providerKey, + 'type' => gettype($providerKey) + ]); + continue; + } + + $provider = $this->providerRegistry->getProviderByKey($providerKey); + if ($provider instanceof BatchInfoProviderInterface) { + $batchProviders[$providerKey] = $provider; + } else { + $regularProviders[$providerKey] = $provider; + } + } + } + + // Process batch providers first (more efficient) + $batchResults = $this->processBatchProviders($parts, $request->fieldMappings, $batchProviders); + + // Process regular providers + $regularResults = $this->processRegularProviders($parts, $request->fieldMappings, $regularProviders, $batchResults); + + // Combine and format results + foreach ($parts as $part) { + $partResult = [ + 'part' => $part, + 'search_results' => [], + 'errors' => [] + ]; + + // Get results from batch and regular processing + $allResults = array_merge( + $batchResults[$part->getId()] ?? [], + $regularResults[$part->getId()] ?? [] + ); + + if (!empty($allResults)) { + $hasAnyResults = true; + $partResult['search_results'] = $this->formatSearchResults($allResults); + } + + $searchResults[] = $partResult; + } + + if (!$hasAnyResults) { + throw new \RuntimeException('No search results found for any of the selected parts'); + } + + return $searchResults; + } + + private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array + { + $batchResults = []; + + foreach ($batchProviders as $providerKey => $provider) { + $keywords = $this->collectKeywordsForProvider($parts, $fieldMappings, $providerKey); + + if (empty($keywords)) { + continue; + } + + try { + $providerResults = $provider->searchByKeywordsBatch($keywords); + + // Map results back to parts + foreach ($parts as $part) { + foreach ($fieldMappings as $mapping) { + if (!in_array($providerKey, $mapping['providers'] ?? [], true)) { + continue; + } + + $keyword = $this->getKeywordFromField($part, $mapping['field']); + if ($keyword && isset($providerResults[$keyword])) { + foreach ($providerResults[$keyword] as $dto) { + $batchResults[$part->getId()][] = new BulkSearchResultDTO( + baseDto: $dto, + sourceField: $mapping['field'], + sourceKeyword: $keyword, + localPart: $this->existingPartFinder->findFirstExisting($dto), + priority: $mapping['priority'] ?? 1 + ); + } + } + } + } + } catch (\Exception $e) { + $this->logger->error('Batch search failed for provider ' . $providerKey, [ + 'error' => $e->getMessage(), + 'provider' => $providerKey + ]); + } + } + + return $batchResults; + } + + private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array + { + $regularResults = []; + + foreach ($parts as $part) { + $regularResults[$part->getId()] = []; + + // Skip if we already have batch results for this part + if (!empty($excludeResults[$part->getId()] ?? [])) { + continue; + } + + foreach ($fieldMappings as $mapping) { + $field = $mapping['field']; + $providers = array_intersect($mapping['providers'] ?? [], array_keys($regularProviders)); + + if (empty($providers)) { + continue; + } + + $keyword = $this->getKeywordFromField($part, $field); + if (!$keyword) { + continue; + } + + try { + $dtos = $this->infoRetriever->searchByKeyword($keyword, $providers); + + foreach ($dtos as $dto) { + $regularResults[$part->getId()][] = new BulkSearchResultDTO( + baseDto: $dto, + sourceField: $field, + sourceKeyword: $keyword, + localPart: $this->existingPartFinder->findFirstExisting($dto), + priority: $mapping['priority'] ?? 1 + ); + } + } catch (ClientException $e) { + $this->logger->error('Regular search failed', [ + 'part_id' => $part->getId(), + 'field' => $field, + 'error' => $e->getMessage() + ]); + } + } + } + + return $regularResults; + } + + private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array + { + $keywords = []; + + foreach ($parts as $part) { + foreach ($fieldMappings as $mapping) { + if (!in_array($providerKey, $mapping['providers'] ?? [], true)) { + continue; + } + + $keyword = $this->getKeywordFromField($part, $mapping['field']); + if ($keyword && !in_array($keyword, $keywords, true)) { + $keywords[] = $keyword; + } + } + } + + return $keywords; + } + + private function getKeywordFromField(Part $part, string $field): ?string + { + return match ($field) { + 'mpn' => $part->getManufacturerProductNumber(), + 'name' => $part->getName(), + default => $this->getSupplierPartNumber($part, $field) + }; + } + + private function getSupplierPartNumber(Part $part, string $field): ?string + { + if (!str_ends_with($field, '_spn')) { + return null; + } + + $supplierKey = substr($field, 0, -4); + $suppliers = $this->entityManager->getRepository(Supplier::class)->findAll(); + + foreach ($suppliers as $supplier) { + $normalizedName = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName())); + if ($normalizedName === $supplierKey) { + $orderDetail = $part->getOrderdetails()->filter( + fn($od) => $od->getSupplier()?->getId() === $supplier->getId() + )->first(); + + return $orderDetail ? $orderDetail->getSupplierpartnr() : null; + } + } + + return null; + } + + private function formatSearchResults(array $bulkResults): array + { + // Sort by priority and remove duplicates + usort($bulkResults, fn($a, $b) => $a->priority <=> $b->priority); + + $uniqueResults = []; + $seenKeys = []; + + foreach ($bulkResults as $result) { + $key = "{$result->provider_key}|{$result->provider_id}"; + if (!in_array($key, $seenKeys, true)) { + $seenKeys[] = $key; + $uniqueResults[] = [ + 'dto' => $result, + 'localPart' => $result->localPart, + 'source_field' => $result->sourceField, + 'source_keyword' => $result->sourceKeyword + ]; + } + } + + return $uniqueResults; + } + + public function prefetchDetailsForResults(array $searchResults): void + { + $prefetchCount = 0; + + foreach ($searchResults as $partResult) { + foreach ($partResult['search_results'] as $result) { + $dto = $result['dto']; + + try { + $this->infoRetriever->getDetails($dto->provider_key, $dto->provider_id); + $prefetchCount++; + } catch (\Exception $e) { + $this->logger->warning('Failed to prefetch details for provider part', [ + 'provider_key' => $dto->provider_key, + 'provider_id' => $dto->provider_id, + 'error' => $e->getMessage() + ]); + } + } + } + + $this->logger->info("Prefetched details for {$prefetchCount} search results"); + } +} \ No newline at end of file diff --git a/src/Services/InfoProviderSystem/DTOs/BulkSearchRequestDTO.php b/src/Services/InfoProviderSystem/DTOs/BulkSearchRequestDTO.php new file mode 100644 index 00000000..55e1f65a --- /dev/null +++ b/src/Services/InfoProviderSystem/DTOs/BulkSearchRequestDTO.php @@ -0,0 +1,14 @@ +provider_key, + provider_id: $baseDto->provider_id, + name: $baseDto->name, + description: $baseDto->description, + category: $baseDto->category, + manufacturer: $baseDto->manufacturer, + mpn: $baseDto->mpn, + preview_image_url: $baseDto->preview_image_url, + manufacturing_status: $baseDto->manufacturing_status, + provider_url: $baseDto->provider_url, + footprint: $baseDto->footprint + ); + } +} \ No newline at end of file diff --git a/templates/info_providers/bulk_import/manage.html.twig b/templates/info_providers/bulk_import/manage.html.twig index 0a21f211..691bf7f3 100644 --- a/templates/info_providers/bulk_import/manage.html.twig +++ b/templates/info_providers/bulk_import/manage.html.twig @@ -10,6 +10,12 @@ {% block card_content %} +
+

{% trans %}info_providers.bulk_import.manage_jobs_description{% endtrans %} @@ -89,12 +95,12 @@ {% endif %} {% if job.canBeStopped %} - {% endif %} {% if job.isCompleted or job.isFailed or job.isStopped %} - {% endif %} @@ -115,54 +121,6 @@

{% endif %} -{% endblock %} +
-{% block scripts %} - {% endblock %} \ No newline at end of file diff --git a/templates/info_providers/bulk_import/step1.html.twig b/templates/info_providers/bulk_import/step1.html.twig index af6a2fcb..16880082 100644 --- a/templates/info_providers/bulk_import/step1.html.twig +++ b/templates/info_providers/bulk_import/step1.html.twig @@ -14,6 +14,8 @@ {% block card_content %} +
+ {% if existing_jobs is not empty %}
@@ -134,7 +136,12 @@ {{ form_start(form) }} -
+
{% trans %}info_providers.bulk_import.field_mappings{% endtrans %}
{% trans %}info_providers.bulk_import.field_mappings_help{% endtrans %} @@ -150,14 +157,14 @@ {% trans %}info_providers.bulk_import.actions.label{% endtrans %} - + {% for mapping in form.field_mappings %} {{ form_widget(mapping.field) }}{{ form_errors(mapping.field) }} {{ form_widget(mapping.providers) }}{{ form_errors(mapping.providers) }} {{ form_widget(mapping.priority) }}{{ form_errors(mapping.priority) }} - @@ -165,7 +172,7 @@ {% endfor %} -
@@ -185,7 +192,7 @@ {{ form_help(form.prefetch_details) }}
- {{ form_widget(form.submit) }} + {{ form_widget(form.submit, {'attr': {'class': 'btn btn-primary', 'data-field-mapping-target': 'submitButton'}}) }}
{{ form_end(form) }} @@ -291,140 +298,7 @@ {% endfor %} {% endif %} -{% endblock %} - -{% block scripts %} - +
+ {% endblock %} diff --git a/templates/info_providers/bulk_import/step2.html.twig b/templates/info_providers/bulk_import/step2.html.twig index f2ab8ad7..3a812abf 100644 --- a/templates/info_providers/bulk_import/step2.html.twig +++ b/templates/info_providers/bulk_import/step2.html.twig @@ -14,6 +14,14 @@ {% block card_content %} +
{{ job.displayNameKey|trans(job.displayNameParams) }} - {{ job.formattedTimestamp }}
@@ -41,10 +49,10 @@
Progress
- {{ job.completedPartsCount }} / {{ job.partCount }} completed + {{ job.completedPartsCount }} / {{ job.partCount }} completed
-
@@ -72,12 +80,32 @@
+ +
+
+
+
+
{% trans %}info_providers.bulk_import.research.title{% endtrans %}
+ {% trans %}info_providers.bulk_import.research.description{% endtrans %} +
+
+ +
+
+
+
+ {% for part_result in search_results %} {% set part = part_result.part %} {% set isCompleted = job.isPartCompleted(part.id) %} {% set isSkipped = job.isPartSkipped(part.id) %}
@@ -101,19 +129,26 @@
+ {% if not isCompleted and not isSkipped %} - - {% elseif isCompleted %} - {% elseif isSkipped %} - {% endif %} @@ -172,7 +207,7 @@ {{ result.source_field ?? 'unknown' }} {% if result.source_keyword %}
{{ result.source_keyword }} - {% endif %} + {% endif %}
@@ -197,152 +232,6 @@
{% endfor %} +
{% endblock %} -{% block scripts %} - -{% endblock %} \ No newline at end of file diff --git a/translations/messages.en.xlf b/translations/messages.en.xlf index 1f47709f..fb3f66a7 100644 --- a/translations/messages.en.xlf +++ b/translations/messages.en.xlf @@ -13949,5 +13949,53 @@ Please note, that you can not impersonate a disabled user. If you try you will g Example: Priority 1: "LCSC SPN → LCSC", Priority 2: "MPN → LCSC + Mouser", Priority 3: "Name → All providers" + + + info_providers.bulk_import.search.submit + Search Providers + + + + + info_providers.bulk_import.searching + Searching + + + + + info_providers.bulk_import.research.title + Research Parts + + + + + info_providers.bulk_import.research.description + Re-search for parts using updated information (e.g., new MPNs). Uses the same field mappings as the original search. + + + + + info_providers.bulk_import.research.all_pending + Research All Pending Parts + + + + + info_providers.bulk_import.research.part + Research + + + + + info_providers.bulk_import.research.part_tooltip + Research this part with updated information + + + + + info_providers.bulk_import.max_mappings_reached + Maximum number of mappings reached + +