diff --git a/.gitignore b/.gitignore index 76655919..dd5c43db 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,6 @@ yarn-error.log ###> phpstan/phpstan ### phpstan.neon ###< phpstan/phpstan ### + +.claude/ +CLAUDE.md \ No newline at end of file diff --git a/assets/controllers/field_mapping_controller.js b/assets/controllers/field_mapping_controller.js index 738035c2..9c9c8ac6 100644 --- a/assets/controllers/field_mapping_controller.js +++ b/assets/controllers/field_mapping_controller.js @@ -22,10 +22,8 @@ export default class extends Controller { select.addEventListener('change', this.updateFieldOptions.bind(this)) }) - // Add click listener to add button - if (this.hasAddButtonTarget) { - this.addButtonTarget.addEventListener('click', this.addMapping.bind(this)) - } + // Note: Add button click is handled by Stimulus action in template (data-action="click->field-mapping#addMapping") + // No manual event listener needed // Form submit handler const form = this.element.querySelector('form') @@ -36,20 +34,20 @@ export default class extends Controller { addMapping() { const currentMappings = this.tbodyTarget.querySelectorAll('.mapping-row').length - + if (currentMappings >= this.maxMappingsValue) { alert(this.maxMappingsReachedMessageValue) return } - + const newRowHtml = this.prototypeValue.replace(/__name__/g, this.mappingIndexValue) const tempDiv = document.createElement('div') tempDiv.innerHTML = newRowHtml - + const fieldWidget = tempDiv.querySelector('select[name*="[field]"]') || tempDiv.children[0] const providerWidget = tempDiv.querySelector('select[name*="[providers]"]') || tempDiv.children[1] const priorityWidget = tempDiv.querySelector('input[name*="[priority]"]') || tempDiv.children[2] - + const newRow = document.createElement('tr') newRow.className = 'mapping-row' newRow.innerHTML = ` @@ -62,16 +60,16 @@ export default class extends Controller { ` - + this.tbodyTarget.appendChild(newRow) this.mappingIndexValue++ - + const newFieldSelect = newRow.querySelector('select[name*="[field]"]') if (newFieldSelect) { newFieldSelect.value = '' newFieldSelect.addEventListener('change', this.updateFieldOptions.bind(this)) } - + this.updateFieldOptions() this.updateAddButtonState() } diff --git a/config/parameters.yaml b/config/parameters.yaml index 154fbd8a..5b40899d 100644 --- a/config/parameters.yaml +++ b/config/parameters.yaml @@ -104,3 +104,9 @@ parameters: env(SAML_ROLE_MAPPING): '{}' env(DATABASE_EMULATE_NATURAL_SORT): 0 + + ###################################################################################################################### + # Bulk Info Provider Import Configuration + ###################################################################################################################### + partdb.bulk_import.batch_size: 20 # Number of parts to process in each batch during bulk operations + partdb.bulk_import.max_parts_per_operation: 1000 # Maximum number of parts allowed per bulk import operation diff --git a/config/services.yaml b/config/services.yaml index 17611cea..d0769cd1 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -17,6 +17,8 @@ services: bool $gdpr_compliance: '%partdb.gdpr_compliance%' bool $kernel_debug_enabled: '%kernel.debug%' string $kernel_cache_dir: '%kernel.cache_dir%' + int $bulkImportBatchSize: '%partdb.bulk_import.batch_size%' + int $bulkImportMaxParts: '%partdb.bulk_import.max_parts_per_operation%' _instanceof: App\Services\LabelSystem\PlaceholderProviders\PlaceholderProviderInterface: diff --git a/src/Controller/BulkInfoProviderImportController.php b/src/Controller/BulkInfoProviderImportController.php index 7675b010..11b6f767 100644 --- a/src/Controller/BulkInfoProviderImportController.php +++ b/src/Controller/BulkInfoProviderImportController.php @@ -29,7 +29,7 @@ use App\Entity\Parts\Part; use App\Entity\Parts\Supplier; use App\Form\InfoProviderSystem\GlobalFieldMappingType; use App\Services\InfoProviderSystem\BulkInfoProviderService; -use App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO; +use App\Services\InfoProviderSystem\DTOs\FieldMappingDTO; use Doctrine\ORM\EntityManagerInterface; use Psr\Log\LoggerInterface; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; @@ -45,10 +45,27 @@ class BulkInfoProviderImportController extends AbstractController public function __construct( private readonly BulkInfoProviderService $bulkService, private readonly EntityManagerInterface $entityManager, - private readonly LoggerInterface $logger + private readonly LoggerInterface $logger, + private readonly int $bulkImportBatchSize, + private readonly int $bulkImportMaxParts ) { } + /** + * Convert field mappings from array format to FieldMappingDTO[]. + * + * @param array $fieldMappings Array of field mapping arrays + * @return FieldMappingDTO[] Array of FieldMappingDTO objects + */ + private function convertFieldMappingsToDto(array $fieldMappings): array + { + $dtos = []; + foreach ($fieldMappings as $mapping) { + $dtos[] = FieldMappingDTO::fromArray($mapping); + } + return $dtos; + } + private function createErrorResponse(string $message, int $statusCode = 400, array $context = []): JsonResponse { $this->logger->warning('Bulk import operation failed', array_merge([ @@ -122,7 +139,17 @@ class BulkInfoProviderImportController extends AbstractController return $this->redirectToRoute('homepage'); } - if (count($parts) > 50) { + // Validate against configured maximum + if (count($parts) > $this->bulkImportMaxParts) { + $this->addFlash('error', sprintf( + 'Too many parts selected (%d). Maximum allowed is %d parts per operation.', + count($parts), + $this->bulkImportMaxParts + )); + return $this->redirectToRoute('homepage'); + } + + if (count($parts) > ($this->bulkImportMaxParts / 2)) { $this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.'); } @@ -164,6 +191,13 @@ class BulkInfoProviderImportController extends AbstractController throw new \RuntimeException('User must be authenticated and of type User'); } + // Validate part count against configuration limit + if (count($parts) > $this->bulkImportMaxParts) { + $this->addFlash('error', "Too many parts selected. Maximum allowed: {$this->bulkImportMaxParts}"); + $partIds = array_map(fn($part) => $part->getId(), $parts); + return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]); + } + // Create and save the job $job = new BulkInfoProviderImportJob(); $job->setFieldMappings($fieldMappings); @@ -179,13 +213,11 @@ class BulkInfoProviderImportController extends AbstractController $this->entityManager->flush(); try { - $searchRequest = new BulkSearchRequestDTO( - fieldMappings: $fieldMappings, - prefetchDetails: $prefetchDetails, - parts: $parts - ); + $fieldMappingDtos = $this->convertFieldMappingsToDto($fieldMappings); + $searchResultsDto = $this->bulkService->performBulkSearch($parts, $fieldMappingDtos, $prefetchDetails); - $searchResults = $this->bulkService->performBulkSearch($searchRequest); + // Convert DTO back to array format for legacy compatibility + $searchResults = $searchResultsDto->toArray(); // Save search results to job $job->setSearchResults($job->serializeSearchResults($searchResults)); @@ -210,6 +242,7 @@ class BulkInfoProviderImportController extends AbstractController $this->entityManager->flush(); $this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage()); + $partIds = array_map(fn($part) => $part->getId(), $parts); return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]); } } @@ -441,14 +474,11 @@ class BulkInfoProviderImportController extends AbstractController $fieldMappings = $job->getFieldMappings(); $prefetchDetails = $job->isPrefetchDetails(); - $searchRequest = new BulkSearchRequestDTO( - fieldMappings: $fieldMappings, - prefetchDetails: $prefetchDetails, - parts: [$part] - ); + $fieldMappingDtos = $this->convertFieldMappingsToDto($fieldMappings); try { - $searchResults = $this->bulkService->performBulkSearch($searchRequest); + $searchResultsDto = $this->bulkService->performBulkSearch([$part], $fieldMappingDtos, $prefetchDetails); + $searchResults = $searchResultsDto->toArray(); } catch (\Exception $searchException) { // Handle "no search results found" as a normal case, not an error if (str_contains($searchException->getMessage(), 'No search results found')) { @@ -500,13 +530,11 @@ class BulkInfoProviderImportController extends AbstractController return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]); } - // Get all part IDs that are not completed or skipped + // Get all parts that are not completed or skipped $parts = []; - $partIds = []; foreach ($job->getJobParts() as $jobPart) { if (!$jobPart->isCompleted() && !$jobPart->isSkipped()) { $parts[] = $jobPart->getPart(); - $partsIds[] = $jobPart->getPart()->getId(); } } @@ -520,26 +548,22 @@ class BulkInfoProviderImportController extends AbstractController try { $fieldMappings = $job->getFieldMappings(); + $fieldMappingDtos = $this->convertFieldMappingsToDto($fieldMappings); $prefetchDetails = $job->isPrefetchDetails(); // Process in batches to reduce memory usage for large operations - $batchSize = 20; // Configurable batch size for memory management $allResults = []; - $batches = array_chunk($parts, $batchSize); + $batches = array_chunk($parts, $this->bulkImportBatchSize); foreach ($batches as $batch) { - $searchRequest = new BulkSearchRequestDTO( - fieldMappings: $fieldMappings, - prefetchDetails: $prefetchDetails, - parts: $batch - ); - - $batchResults = $this->bulkService->performBulkSearch($searchRequest); + $batchResultsDto = $this->bulkService->performBulkSearch($batch, $fieldMappingDtos, $prefetchDetails); + $batchResults = $batchResultsDto->toArray(); $allResults = array_merge($allResults, $batchResults); - // Clear entity manager periodically to prevent memory issues + // Properly manage entity manager memory without losing state + $jobId = $job->getId(); $this->entityManager->clear(); - $job = $this->entityManager->find(BulkInfoProviderImportJob::class, $job->getId()); + $job = $this->entityManager->find(BulkInfoProviderImportJob::class, $jobId); } // Update the job's search results @@ -564,7 +588,7 @@ class BulkInfoProviderImportController extends AbstractController 500, [ 'job_id' => $jobId, - 'part_ids' => $partsIds, + 'part_count' => count($parts), 'exception' => $e->getMessage() ] ); diff --git a/src/Services/InfoProviderSystem/BulkInfoProviderService.php b/src/Services/InfoProviderSystem/BulkInfoProviderService.php index ec8a4bd3..c2b4dc90 100644 --- a/src/Services/InfoProviderSystem/BulkInfoProviderService.php +++ b/src/Services/InfoProviderSystem/BulkInfoProviderService.php @@ -4,12 +4,14 @@ declare(strict_types=1); namespace App\Services\InfoProviderSystem; -use A\B; -use App\Entity\BulkInfoProviderImportJob; +use App\Entity\Base\AbstractPartsContainingDBElement; use App\Entity\Parts\Part; use App\Entity\Parts\Supplier; -use App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO; use App\Services\InfoProviderSystem\DTOs\BulkSearchResultDTO; +use App\Services\InfoProviderSystem\DTOs\BulkSearchResponseDTO; +use App\Services\InfoProviderSystem\DTOs\FieldMappingDTO; +use App\Services\InfoProviderSystem\DTOs\PartSearchResultDTO; +use App\Services\InfoProviderSystem\DTOs\SearchResultWithMetadataDTO; use App\Services\InfoProviderSystem\Providers\BatchInfoProviderInterface; use App\Services\InfoProviderSystem\Providers\InfoProviderInterface; use Doctrine\ORM\EntityManagerInterface; @@ -18,6 +20,9 @@ use Symfony\Component\HttpClient\Exception\ClientException; final class BulkInfoProviderService { + /** @var array Cache for normalized supplier names */ + private array $supplierCache = []; + public function __construct( private readonly PartInfoRetriever $infoRetriever, private readonly ExistingPartFinder $existingPartFinder, @@ -26,24 +31,31 @@ final class BulkInfoProviderService private readonly LoggerInterface $logger ) {} - public function performBulkSearch(BulkSearchRequestDTO $request): array + /** + * Perform bulk search across multiple parts and providers. + * + * @param Part[] $parts Array of parts to search for + * @param FieldMappingDTO[] $fieldMappings Array of field mappings defining search strategy + * @param bool $prefetchDetails Whether to prefetch detailed information for results + * @return BulkSearchResponseDTO Structured response containing all search results + * @throws \InvalidArgumentException If no valid parts provided + * @throws \RuntimeException If no search results found for any parts + */ + public function performBulkSearch(array $parts, array $fieldMappings, bool $prefetchDetails = false): BulkSearchResponseDTO { - $parts = $request->parts; - if (empty($parts)) { throw new \InvalidArgumentException('No valid parts found for bulk import'); } - $searchResults = []; + $partResults = []; $hasAnyResults = false; // Group providers by batch capability $batchProviders = []; $regularProviders = []; - foreach ($request->fieldMappings as $mapping) { - $providers = $mapping['providers'] ?? []; - foreach ($providers as $providerKey) { + foreach ($fieldMappings as $mapping) { + foreach ($mapping->providers as $providerKey) { if (!is_string($providerKey)) { $this->logger->error('Invalid provider key type', [ 'providerKey' => $providerKey, @@ -62,18 +74,14 @@ final class BulkInfoProviderService } // Process batch providers first (more efficient) - $batchResults = $this->processBatchProviders($parts, $request->fieldMappings, $batchProviders); + $batchResults = $this->processBatchProviders($parts, $fieldMappings, $batchProviders); // Process regular providers - $regularResults = $this->processRegularProviders($parts, $request->fieldMappings, $regularProviders, $batchResults); + $regularResults = $this->processRegularProviders($parts, $fieldMappings, $regularProviders, $batchResults); - // Combine and format results + // Combine and format results for each part foreach ($parts as $part) { - $partResult = [ - 'part' => $part, - 'search_results' => [], - 'errors' => [] - ]; + $searchResults = []; // Get results from batch and regular processing $allResults = array_merge( @@ -83,24 +91,37 @@ final class BulkInfoProviderService if (!empty($allResults)) { $hasAnyResults = true; - $partResult['search_results'] = $this->formatSearchResults($allResults); + $searchResults = $this->formatSearchResults($allResults); } - $searchResults[] = $partResult; + $partResults[] = new PartSearchResultDTO( + part: $part, + searchResults: $searchResults, + errors: [] + ); } if (!$hasAnyResults) { throw new \RuntimeException('No search results found for any of the selected parts'); } - return $searchResults; + $response = new BulkSearchResponseDTO($partResults); + + // Prefetch details if requested + if ($prefetchDetails) { + $this->prefetchDetailsForResults($response); + } + + return $response; } /** - * @param Part[] $parts - * @param array $fieldMappings - * @param array $batchProviders - * @return array A list of results indexed by part ID + * Process parts using batch-capable info providers. + * + * @param Part[] $parts Array of parts to search for + * @param FieldMappingDTO[] $fieldMappings Array of field mapping configurations + * @param array $batchProviders Batch providers indexed by key + * @return array Results indexed by part ID */ private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array { @@ -119,19 +140,19 @@ final class BulkInfoProviderService // Map results back to parts foreach ($parts as $part) { foreach ($fieldMappings as $mapping) { - if (!in_array($providerKey, $mapping['providers'] ?? [], true)) { + if (!in_array($providerKey, $mapping->providers, true)) { continue; } - $keyword = $this->getKeywordFromField($part, $mapping['field']); + $keyword = $this->getKeywordFromField($part, $mapping->field); if ($keyword && isset($providerResults[$keyword])) { foreach ($providerResults[$keyword] as $dto) { $batchResults[$part->getId()][] = new BulkSearchResultDTO( baseDto: $dto, - sourceField: $mapping['field'], + sourceField: $mapping->field, sourceKeyword: $keyword, localPart: $this->existingPartFinder->findFirstExisting($dto), - priority: $mapping['priority'] ?? 1 + priority: $mapping->priority ); } } @@ -149,11 +170,13 @@ final class BulkInfoProviderService } /** - * @param Part[] $parts - * @param array $fieldMappings - * @param array $regularProviders The info providers that do not support batch searching, indexed by their provider key - * @param array $excludeResults - * @return array A list of results indexed by part ID + * Process parts using regular (non-batch) info providers. + * + * @param Part[] $parts Array of parts to search for + * @param FieldMappingDTO[] $fieldMappings Array of field mapping configurations + * @param array $regularProviders Regular providers indexed by key + * @param array $excludeResults Results to exclude (from batch processing) + * @return array Results indexed by part ID */ private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array { @@ -168,14 +191,13 @@ final class BulkInfoProviderService } foreach ($fieldMappings as $mapping) { - $field = $mapping['field']; - $providers = array_intersect($mapping['providers'] ?? [], array_keys($regularProviders)); + $providers = array_intersect($mapping->providers, array_keys($regularProviders)); if (empty($providers)) { continue; } - $keyword = $this->getKeywordFromField($part, $field); + $keyword = $this->getKeywordFromField($part, $mapping->field); if (!$keyword) { continue; } @@ -186,16 +208,16 @@ final class BulkInfoProviderService foreach ($dtos as $dto) { $regularResults[$part->getId()][] = new BulkSearchResultDTO( baseDto: $dto, - sourceField: $field, + sourceField: $mapping->field, sourceKeyword: $keyword, localPart: $this->existingPartFinder->findFirstExisting($dto), - priority: $mapping['priority'] ?? 1 + priority: $mapping->priority ); } } catch (ClientException $e) { $this->logger->error('Regular search failed', [ 'part_id' => $part->getId(), - 'field' => $field, + 'field' => $mapping->field, 'error' => $e->getMessage() ]); } @@ -206,10 +228,12 @@ final class BulkInfoProviderService } /** - * @param Part[] $parts - * @param array $fieldMappings - * @param string $providerKey - * @return string[] + * Collect unique keywords for a specific provider from all parts and field mappings. + * + * @param Part[] $parts Array of parts to collect keywords from + * @param FieldMappingDTO[] $fieldMappings Array of field mapping configurations + * @param string $providerKey The provider key to collect keywords for + * @return string[] Array of unique keywords */ private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array { @@ -217,11 +241,11 @@ final class BulkInfoProviderService foreach ($parts as $part) { foreach ($fieldMappings as $mapping) { - if (!in_array($providerKey, $mapping['providers'] ?? [], true)) { + if (!in_array($providerKey, $mapping->providers, true)) { continue; } - $keyword = $this->getKeywordFromField($part, $mapping['field']); + $keyword = $this->getKeywordFromField($part, $mapping->field); if ($keyword && !in_array($keyword, $keywords, true)) { $keywords[] = $keyword; } @@ -247,22 +271,66 @@ final class BulkInfoProviderService } $supplierKey = substr($field, 0, -4); + $supplier = $this->getSupplierByNormalizedName($supplierKey); + + if (!$supplier) { + return null; + } + + $orderDetail = $part->getOrderdetails()->filter( + fn($od) => $od->getSupplier()?->getId() === $supplier->getId() + )->first(); + + return $orderDetail !== false ? $orderDetail->getSupplierpartnr() : null; + } + + /** + * Get supplier by normalized name with caching to prevent N+1 queries. + * + * @param string $normalizedKey The normalized supplier key to search for + * @return Supplier|null The matching supplier or null if not found + */ + private function getSupplierByNormalizedName(string $normalizedKey): ?Supplier + { + // Check cache first + if (isset($this->supplierCache[$normalizedKey])) { + return $this->supplierCache[$normalizedKey]; + } + + // Use efficient database query with PHP normalization + // Since DQL doesn't support REPLACE, we'll load all suppliers once and cache the normalization + if (empty($this->supplierCache)) { + $this->loadSuppliersIntoCache(); + } + + $supplier = $this->supplierCache[$normalizedKey] ?? null; + + // Cache the result (including null results to prevent repeated queries) + $this->supplierCache[$normalizedKey] = $supplier; + + return $supplier; + } + + /** + * Load all suppliers into cache with normalized names to avoid N+1 queries. + */ + private function loadSuppliersIntoCache(): void + { + /** @var Supplier[] $suppliers */ $suppliers = $this->entityManager->getRepository(Supplier::class)->findAll(); foreach ($suppliers as $supplier) { $normalizedName = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName())); - if ($normalizedName === $supplierKey) { - $orderDetail = $part->getOrderdetails()->filter( - fn($od) => $od->getSupplier()?->getId() === $supplier->getId() - )->first(); - - return $orderDetail ? $orderDetail->getSupplierpartnr() : null; - } + $this->supplierCache[$normalizedName] = $supplier; } - - return null; } + /** + * Format and deduplicate search results. + * + * @param BulkSearchResultDTO[] $bulkResults Array of bulk search results + * @return SearchResultWithMetadataDTO[] Array of formatted search results with metadata + */ private function formatSearchResults(array $bulkResults): array { // Sort by priority and remove duplicates @@ -272,38 +340,64 @@ final class BulkInfoProviderService $seenKeys = []; foreach ($bulkResults as $result) { - $key = "{$result->provider_key}|{$result->provider_id}"; + $key = "{$result->getProviderKey()}|{$result->getProviderId()}"; if (!in_array($key, $seenKeys, true)) { $seenKeys[] = $key; - $uniqueResults[] = [ - 'dto' => $result, - 'localPart' => $result->localPart, - 'source_field' => $result->sourceField, - 'source_keyword' => $result->sourceKeyword - ]; + $uniqueResults[] = new SearchResultWithMetadataDTO( + searchResult: $result, + localPart: $result->localPart, + sourceField: $result->sourceField, + sourceKeyword: $result->sourceKeyword + ); } } return $uniqueResults; } - public function prefetchDetailsForResults(array $searchResults): void + /** + * Prefetch detailed information for search results. + * + * @param BulkSearchResponseDTO|array $searchResults Search results (supports both new DTO and legacy array format) + */ + public function prefetchDetailsForResults($searchResults): void { $prefetchCount = 0; - foreach ($searchResults as $partResult) { - foreach ($partResult['search_results'] as $result) { - $dto = $result['dto']; + // Handle both new DTO format and legacy array format for backwards compatibility + if ($searchResults instanceof BulkSearchResponseDTO) { + foreach ($searchResults->partResults as $partResult) { + foreach ($partResult->searchResults as $result) { + $dto = $result->searchResult; - try { - $this->infoRetriever->getDetails($dto->provider_key, $dto->provider_id); - $prefetchCount++; - } catch (\Exception $e) { - $this->logger->warning('Failed to prefetch details for provider part', [ - 'provider_key' => $dto->provider_key, - 'provider_id' => $dto->provider_id, - 'error' => $e->getMessage() - ]); + try { + $this->infoRetriever->getDetails($dto->getProviderKey(), $dto->getProviderId()); + $prefetchCount++; + } catch (\Exception $e) { + $this->logger->warning('Failed to prefetch details for provider part', [ + 'provider_key' => $dto->getProviderKey(), + 'provider_id' => $dto->getProviderId(), + 'error' => $e->getMessage() + ]); + } + } + } + } else { + // Legacy array format support + foreach ($searchResults as $partResult) { + foreach ($partResult['search_results'] as $result) { + $dto = $result['dto']; + + try { + $this->infoRetriever->getDetails($dto->getProviderKey(), $dto->getProviderId()); + $prefetchCount++; + } catch (\Exception $e) { + $this->logger->warning('Failed to prefetch details for provider part', [ + 'provider_key' => $dto->getProviderKey(), + 'provider_id' => $dto->getProviderId(), + 'error' => $e->getMessage() + ]); + } } } } diff --git a/src/Services/InfoProviderSystem/DTOs/BulkSearchRequestDTO.php b/src/Services/InfoProviderSystem/DTOs/BulkSearchRequestDTO.php deleted file mode 100644 index c915c3be..00000000 --- a/src/Services/InfoProviderSystem/DTOs/BulkSearchRequestDTO.php +++ /dev/null @@ -1,21 +0,0 @@ -. + */ + +declare(strict_types=1); + +namespace App\Services\InfoProviderSystem\DTOs; + +/** + * Represents the complete response from a bulk info provider search operation. + * This DTO provides type safety and clear structure instead of complex arrays. + */ +readonly class BulkSearchResponseDTO +{ + /** + * @param PartSearchResultDTO[] $partResults Array of search results for each part + */ + public function __construct( + public array $partResults + ) {} + + /** + * Create from legacy array format for backwards compatibility. + * @param array $data Array of part result arrays in legacy format + */ + public static function fromArray(array $data): self + { + $partResults = []; + foreach ($data as $partData) { + $partResults[] = PartSearchResultDTO::fromArray($partData); + } + + return new self($partResults); + } + + /** + * Convert to legacy array format for backwards compatibility. + */ + public function toArray(): array + { + $result = []; + foreach ($this->partResults as $partResult) { + $result[] = $partResult->toArray(); + } + return $result; + } + + /** + * Check if any parts have search results. + */ + public function hasAnyResults(): bool + { + foreach ($this->partResults as $partResult) { + if ($partResult->hasResults()) { + return true; + } + } + return false; + } + + /** + * Get the total number of search results across all parts. + */ + public function getTotalResultCount(): int + { + $count = 0; + foreach ($this->partResults as $partResult) { + $count += $partResult->getResultCount(); + } + return $count; + } + + /** + * Get all parts that have search results. + * @return PartSearchResultDTO[] + */ + public function getPartsWithResults(): array + { + return array_filter($this->partResults, fn($result) => $result->hasResults()); + } + + /** + * Get all parts that have errors. + * @return PartSearchResultDTO[] + */ + public function getPartsWithErrors(): array + { + return array_filter($this->partResults, fn($result) => $result->hasErrors()); + } + + /** + * Get the number of parts processed. + */ + public function getPartCount(): int + { + return count($this->partResults); + } + + /** + * Get the number of parts with successful results. + */ + public function getSuccessfulPartCount(): int + { + return count($this->getPartsWithResults()); + } +} \ No newline at end of file diff --git a/src/Services/InfoProviderSystem/DTOs/BulkSearchResultDTO.php b/src/Services/InfoProviderSystem/DTOs/BulkSearchResultDTO.php index 3f0961c7..7beb4291 100644 --- a/src/Services/InfoProviderSystem/DTOs/BulkSearchResultDTO.php +++ b/src/Services/InfoProviderSystem/DTOs/BulkSearchResultDTO.php @@ -1,32 +1,139 @@ . + */ declare(strict_types=1); namespace App\Services\InfoProviderSystem\DTOs; +use App\Entity\Parts\ManufacturingStatus; use App\Entity\Parts\Part; -class BulkSearchResultDTO extends SearchResultDTO +/** + * Represents a search result from bulk search with additional context information. + * Uses composition instead of inheritance for better maintainability. + */ +readonly class BulkSearchResultDTO { public function __construct( - SearchResultDTO $baseDto, - public readonly ?string $sourceField = null, - public readonly ?string $sourceKeyword = null, - public readonly ?Part $localPart = null, - public readonly int $priority = 1 - ) { - parent::__construct( - provider_key: $baseDto->provider_key, - provider_id: $baseDto->provider_id, - name: $baseDto->name, - description: $baseDto->description, - category: $baseDto->category, - manufacturer: $baseDto->manufacturer, - mpn: $baseDto->mpn, - preview_image_url: $baseDto->preview_image_url, - manufacturing_status: $baseDto->manufacturing_status, - provider_url: $baseDto->provider_url, - footprint: $baseDto->footprint - ); + /** The base search result DTO containing provider data */ + public SearchResultDTO $baseDto, + /** The field that was used to find this result */ + public ?string $sourceField = null, + /** The actual keyword that was searched for */ + public ?string $sourceKeyword = null, + /** Local part that matches this search result, if any */ + public ?Part $localPart = null, + /** Priority for this search result */ + public int $priority = 1 + ) {} + + // Delegation methods for SearchResultDTO properties + public function getProviderKey(): string + { + return $this->baseDto->provider_key; + } + + public function getProviderId(): string + { + return $this->baseDto->provider_id; + } + + public function getName(): string + { + return $this->baseDto->name; + } + + public function getDescription(): string + { + return $this->baseDto->description; + } + + public function getCategory(): ?string + { + return $this->baseDto->category; + } + + public function getManufacturer(): ?string + { + return $this->baseDto->manufacturer; + } + + public function getMpn(): ?string + { + return $this->baseDto->mpn; + } + + public function getPreviewImageUrl(): ?string + { + return $this->baseDto->preview_image_url; + } + + public function getPreviewImageFile(): ?FileDTO + { + return $this->baseDto->preview_image_file; + } + + public function getManufacturingStatus(): ?ManufacturingStatus + { + return $this->baseDto->manufacturing_status; + } + + public function getProviderUrl(): ?string + { + return $this->baseDto->provider_url; + } + + public function getFootprint(): ?string + { + return $this->baseDto->footprint; + } + + // Backwards compatibility properties for legacy code + public function __get(string $name): mixed + { + return match ($name) { + 'provider_key' => $this->baseDto->provider_key, + 'provider_id' => $this->baseDto->provider_id, + 'name' => $this->baseDto->name, + 'description' => $this->baseDto->description, + 'category' => $this->baseDto->category, + 'manufacturer' => $this->baseDto->manufacturer, + 'mpn' => $this->baseDto->mpn, + 'preview_image_url' => $this->baseDto->preview_image_url, + 'preview_image_file' => $this->baseDto->preview_image_file, + 'manufacturing_status' => $this->baseDto->manufacturing_status, + 'provider_url' => $this->baseDto->provider_url, + 'footprint' => $this->baseDto->footprint, + default => throw new \InvalidArgumentException("Property '{$name}' does not exist") + }; + } + + /** + * Magic isset method for backwards compatibility. + */ + public function __isset(string $name): bool + { + return in_array($name, [ + 'provider_key', 'provider_id', 'name', 'description', 'category', + 'manufacturer', 'mpn', 'preview_image_url', 'preview_image_file', + 'manufacturing_status', 'provider_url', 'footprint' + ], true); } } \ No newline at end of file diff --git a/src/Services/InfoProviderSystem/DTOs/FieldMappingDTO.php b/src/Services/InfoProviderSystem/DTOs/FieldMappingDTO.php new file mode 100644 index 00000000..cdbe4bc0 --- /dev/null +++ b/src/Services/InfoProviderSystem/DTOs/FieldMappingDTO.php @@ -0,0 +1,92 @@ +. + */ + +declare(strict_types=1); + +namespace App\Services\InfoProviderSystem\DTOs; + +/** + * Represents a mapping between a part field and the info providers that should search in that field. + * This DTO provides type safety and better structure than raw arrays for field mapping configuration. + */ +readonly class FieldMappingDTO +{ + /** + * @param string $field The field to search in (e.g., 'mpn', 'name', or supplier-specific fields like 'digikey_spn') + * @param string[] $providers Array of provider keys to search with (e.g., ['digikey', 'farnell']) + * @param int $priority Priority for this field mapping (1-10, lower numbers = higher priority) + */ + public function __construct( + public string $field, + public array $providers, + public int $priority = 1 + ) { + if ($priority < 1 || $priority > 10) { + throw new \InvalidArgumentException('Priority must be between 1 and 10'); + } + } + + /** + * Create a FieldMappingDTO from legacy array format. + * @param array{field: string, providers: string[], priority?: int} $data + */ + public static function fromArray(array $data): self + { + return new self( + field: $data['field'], + providers: $data['providers'] ?? [], + priority: $data['priority'] ?? 1 + ); + } + + /** + * Convert this DTO to the legacy array format for backwards compatibility. + * @return array{field: string, providers: string[], priority: int} + */ + public function toArray(): array + { + return [ + 'field' => $this->field, + 'providers' => $this->providers, + 'priority' => $this->priority, + ]; + } + + /** + * Check if this field mapping is for a supplier part number field. + */ + public function isSupplierPartNumberField(): bool + { + return str_ends_with($this->field, '_spn'); + } + + /** + * Get the supplier key from a supplier part number field. + * Returns null if this is not a supplier part number field. + */ + public function getSupplierKey(): ?string + { + if (!$this->isSupplierPartNumberField()) { + return null; + } + + return substr($this->field, 0, -4); + } +} \ No newline at end of file diff --git a/src/Services/InfoProviderSystem/DTOs/PartSearchResultDTO.php b/src/Services/InfoProviderSystem/DTOs/PartSearchResultDTO.php new file mode 100644 index 00000000..f516ec1e --- /dev/null +++ b/src/Services/InfoProviderSystem/DTOs/PartSearchResultDTO.php @@ -0,0 +1,114 @@ +. + */ + +declare(strict_types=1); + +namespace App\Services\InfoProviderSystem\DTOs; + +use App\Entity\Parts\Part; + +/** + * Represents the search results for a single part from bulk info provider search. + * This DTO provides type safety and clear structure for part search results. + */ +readonly class PartSearchResultDTO +{ + /** + * @param Part $part The part that was searched for + * @param SearchResultWithMetadataDTO[] $searchResults Array of search results found for this part + * @param string[] $errors Array of error messages encountered during search + */ + public function __construct( + public Part $part, + public array $searchResults = [], + public array $errors = [] + ) {} + + /** + * Create from legacy array format for backwards compatibility. + * @param array{part: Part, search_results: array, errors: string[]} $data + */ + public static function fromArray(array $data): self + { + $searchResults = []; + foreach ($data['search_results'] as $result) { + $searchResults[] = SearchResultWithMetadataDTO::fromArray($result); + } + + return new self( + part: $data['part'], + searchResults: $searchResults, + errors: $data['errors'] ?? [] + ); + } + + /** + * Convert to legacy array format for backwards compatibility. + * @return array{part: Part, search_results: array, errors: string[]} + */ + public function toArray(): array + { + $searchResults = []; + foreach ($this->searchResults as $result) { + $searchResults[] = $result->toArray(); + } + + return [ + 'part' => $this->part, + 'search_results' => $searchResults, + 'errors' => $this->errors, + ]; + } + + /** + * Check if this part has any search results. + */ + public function hasResults(): bool + { + return !empty($this->searchResults); + } + + /** + * Check if this part has any errors. + */ + public function hasErrors(): bool + { + return !empty($this->errors); + } + + /** + * Get the number of search results for this part. + */ + public function getResultCount(): int + { + return count($this->searchResults); + } + + /** + * Get search results sorted by priority (ascending). + * @return SearchResultWithMetadataDTO[] + */ + public function getResultsSortedByPriority(): array + { + $results = $this->searchResults; + usort($results, fn($a, $b) => $a->getPriority() <=> $b->getPriority()); + return $results; + } +} \ No newline at end of file diff --git a/src/Services/InfoProviderSystem/DTOs/SearchResultWithMetadataDTO.php b/src/Services/InfoProviderSystem/DTOs/SearchResultWithMetadataDTO.php new file mode 100644 index 00000000..555e3b61 --- /dev/null +++ b/src/Services/InfoProviderSystem/DTOs/SearchResultWithMetadataDTO.php @@ -0,0 +1,95 @@ +. + */ + +declare(strict_types=1); + +namespace App\Services\InfoProviderSystem\DTOs; + +use App\Entity\Parts\Part; + +/** + * Represents a search result with additional metadata about how it was found. + * This DTO encapsulates both the search result data and the context of the search. + */ +readonly class SearchResultWithMetadataDTO +{ + public function __construct( + /** The search result DTO containing part information from the provider */ + public BulkSearchResultDTO $searchResult, + /** Local part that matches this search result, if any */ + public ?Part $localPart = null, + /** The field that was used to find this result (e.g., 'mpn', 'name') */ + public ?string $sourceField = null, + /** The actual keyword/value that was searched for */ + public ?string $sourceKeyword = null + ) {} + + /** + * Create from legacy array format for backwards compatibility. + * @param array{dto: BulkSearchResultDTO, localPart: ?Part, source_field: string, source_keyword: string} $data + */ + public static function fromArray(array $data): self + { + return new self( + searchResult: $data['dto'], + localPart: $data['localPart'] ?? null, + sourceField: $data['source_field'] ?? null, + sourceKeyword: $data['source_keyword'] ?? null + ); + } + + /** + * Convert to legacy array format for backwards compatibility. + * @return array{dto: BulkSearchResultDTO, localPart: ?Part, source_field: ?string, source_keyword: ?string} + */ + public function toArray(): array + { + return [ + 'dto' => $this->searchResult, + 'localPart' => $this->localPart, + 'source_field' => $this->sourceField, + 'source_keyword' => $this->sourceKeyword, + ]; + } + + /** + * Get the priority of this search result. + */ + public function getPriority(): int + { + return $this->searchResult->priority; + } + + /** + * Get the provider key from the search result. + */ + public function getProviderKey(): string + { + return $this->searchResult->getProviderKey(); + } + + /** + * Get the provider ID from the search result. + */ + public function getProviderId(): string + { + return $this->searchResult->getProviderId(); + } +} \ No newline at end of file diff --git a/src/Services/InfoProviderSystem/Providers/MouserProvider.php b/src/Services/InfoProviderSystem/Providers/MouserProvider.php index 6639e5c1..a3c83b25 100644 --- a/src/Services/InfoProviderSystem/Providers/MouserProvider.php +++ b/src/Services/InfoProviderSystem/Providers/MouserProvider.php @@ -132,6 +132,15 @@ class MouserProvider implements InfoProviderInterface ], ]); + // Check for API errors before processing response + if ($response->getStatusCode() !== 200) { + throw new \RuntimeException(sprintf( + 'Mouser API returned HTTP %d: %s', + $response->getStatusCode(), + $response->getContent(false) + )); + } + return $this->responseToDTOArray($response); } @@ -169,6 +178,16 @@ class MouserProvider implements InfoProviderInterface ] ], ]); + + // Check for API errors before processing response + if ($response->getStatusCode() !== 200) { + throw new \RuntimeException(sprintf( + 'Mouser API returned HTTP %d: %s', + $response->getStatusCode(), + $response->getContent(false) + )); + } + $tmp = $this->responseToDTOArray($response); //Ensure that we have exactly one result diff --git a/templates/info_providers/bulk_import/step1.html.twig b/templates/info_providers/bulk_import/step1.html.twig index 16880082..bb9bb351 100644 --- a/templates/info_providers/bulk_import/step1.html.twig +++ b/templates/info_providers/bulk_import/step1.html.twig @@ -140,15 +140,14 @@ data-controller="field-mapping" data-field-mapping-mapping-index-value="{{ form.field_mappings|length }}" data-field-mapping-max-mappings-value="{{ fieldChoices|length }}" - data-field-mapping-prototype-value="{{ form_widget(form.field_mappings.vars.prototype)|e('js') }}" + data-field-mapping-prototype-value="{{ form_widget(form.field_mappings.vars.prototype)|e('html_attr') }}" data-field-mapping-max-mappings-reached-message-value="{{ 'info_providers.bulk_import.max_mappings_reached'|trans|e('js') }}">
{% trans %}info_providers.bulk_import.field_mappings{% endtrans %}
{% trans %}info_providers.bulk_import.field_mappings_help{% endtrans %}
-
- +
@@ -171,11 +170,12 @@ {% endfor %} -
{% trans %}info_providers.bulk_search.search_field{% endtrans %}
- -
+ +
diff --git a/tests/Controller/BulkInfoProviderImportControllerTest.php b/tests/Controller/BulkInfoProviderImportControllerTest.php index d20f3dea..f6986dbc 100644 --- a/tests/Controller/BulkInfoProviderImportControllerTest.php +++ b/tests/Controller/BulkInfoProviderImportControllerTest.php @@ -657,20 +657,16 @@ class BulkInfoProviderImportControllerTest extends WebTestCase // Test that the service can extract keywords from parts $bulkService = $client->getContainer()->get(\App\Services\InfoProviderSystem\BulkInfoProviderService::class); - // Create a test request to verify the service works - $request = new \App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO( - fieldMappings: [ - ['field' => 'name', 'providers' => ['test'], 'priority' => 1], - ['field' => 'mpn', 'providers' => ['test'], 'priority' => 2] - ], - prefetchDetails: false, - parts: [$part] - ); + // Create field mappings to verify the service works + $fieldMappings = [ + new \App\Services\InfoProviderSystem\DTOs\FieldMappingDTO('name', ['test'], 1), + new \App\Services\InfoProviderSystem\DTOs\FieldMappingDTO('mpn', ['test'], 2) + ]; // The service may return an empty result or throw when no results are found try { - $result = $bulkService->performBulkSearch($request); - $this->assertIsArray($result); + $result = $bulkService->performBulkSearch([$part], $fieldMappings, false); + $this->assertInstanceOf(\App\Services\InfoProviderSystem\DTOs\BulkSearchResponseDTO::class, $result); } catch (\RuntimeException $e) { $this->assertStringContainsString('No search results found', $e->getMessage()); } @@ -792,19 +788,15 @@ class BulkInfoProviderImportControllerTest extends WebTestCase // Test that the service can handle supplier part number fields $bulkService = $client->getContainer()->get(\App\Services\InfoProviderSystem\BulkInfoProviderService::class); - // Create a test request with supplier SPN field mapping - $request = new \App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO( - fieldMappings: [ - ['field' => 'invalid_field', 'providers' => ['test'], 'priority' => 1], - ['field' => 'test_supplier_spn', 'providers' => ['test'], 'priority' => 2] - ], - prefetchDetails: false, - parts: [$part] - ); + // Create field mappings with supplier SPN field mapping + $fieldMappings = [ + new \App\Services\InfoProviderSystem\DTOs\FieldMappingDTO('invalid_field', ['test'], 1), + new \App\Services\InfoProviderSystem\DTOs\FieldMappingDTO('test_supplier_spn', ['test'], 2) + ]; // The service should be able to process the request and throw an exception when no results are found try { - $bulkService->performBulkSearch($request); + $bulkService->performBulkSearch([$part], $fieldMappings, false); $this->fail('Expected RuntimeException to be thrown when no search results are found'); } catch (\RuntimeException $e) { $this->assertStringContainsString('No search results found', $e->getMessage()); @@ -827,18 +819,14 @@ class BulkInfoProviderImportControllerTest extends WebTestCase // Test that the service can handle batch processing $bulkService = $client->getContainer()->get(\App\Services\InfoProviderSystem\BulkInfoProviderService::class); - // Create a test request with multiple keywords - $request = new \App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO( - fieldMappings: [ - ['field' => 'name', 'providers' => ['lcsc'], 'priority' => 1] - ], - prefetchDetails: false, - parts: [$part] - ); + // Create field mappings with multiple keywords + $fieldMappings = [ + new \App\Services\InfoProviderSystem\DTOs\FieldMappingDTO('name', ['lcsc'], 1) + ]; // The service should be able to process the request and throw an exception when no results are found try { - $bulkService->performBulkSearch($request); + $bulkService->performBulkSearch([$part], $fieldMappings, false); $this->fail('Expected RuntimeException to be thrown when no search results are found'); } catch (\RuntimeException $e) { $this->assertStringContainsString('No search results found', $e->getMessage());