groove/Part-DB-server
1
0
Fork 0
mirror of https://github.com/Part-DB/Part-DB-server.git synced 2025-12-06 02:59:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Refactor bulk info provider: replace complex arrays with DTOs

Browse source 
- Add BulkSearchResponseDTO, FieldMappingDTO for type safety
- Use composition instead of inheritance in BulkSearchResultDTO
- Remove unnecessary BulkSearchRequestDTO
- Fix N+1 queries and API error handling
- Fix Add Mapping button functionality
This commit is contained in:
barisgit 2025-09-19 16:28:40 +02:00
parent 8998b006e0
commit 2c195d9767
15 changed files with 838 additions and 195 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -48,3 +48,6 @@ yarn-error.log
###> phpstan/phpstan ###
phpstan.neon
###< phpstan/phpstan ###
.claude/
CLAUDE.md

20
assets/controllers/field_mapping_controller.js
View file

@ -22,10 +22,8 @@ export default class extends Controller {
select.addEventListener('change', this.updateFieldOptions.bind(this))
})
// Add click listener to add button
if (this.hasAddButtonTarget) {
this.addButtonTarget.addEventListener('click', this.addMapping.bind(this))
}
// Note: Add button click is handled by Stimulus action in template (data-action="click->field-mapping#addMapping")
// No manual event listener needed
// Form submit handler
const form = this.element.querySelector('form')
@ -36,20 +34,20 @@ export default class extends Controller {
addMapping() {
const currentMappings = this.tbodyTarget.querySelectorAll('.mapping-row').length
if (currentMappings >= this.maxMappingsValue) {
alert(this.maxMappingsReachedMessageValue)
return
}
const newRowHtml = this.prototypeValue.replace(/__name__/g, this.mappingIndexValue)
const tempDiv = document.createElement('div')
tempDiv.innerHTML = newRowHtml
const fieldWidget = tempDiv.querySelector('select[name*="[field]"]') || tempDiv.children[0]
const providerWidget = tempDiv.querySelector('select[name*="[providers]"]') || tempDiv.children[1]
const priorityWidget = tempDiv.querySelector('input[name*="[priority]"]') || tempDiv.children[2]
const newRow = document.createElement('tr')
newRow.className = 'mapping-row'
newRow.innerHTML = `
@ -62,16 +60,16 @@ export default class extends Controller {
</button>
</td>
`
this.tbodyTarget.appendChild(newRow)
this.mappingIndexValue++
const newFieldSelect = newRow.querySelector('select[name*="[field]"]')
if (newFieldSelect) {
newFieldSelect.value = ''
newFieldSelect.addEventListener('change', this.updateFieldOptions.bind(this))
}
this.updateFieldOptions()
this.updateAddButtonState()
}

6
config/parameters.yaml
View file

@ -104,3 +104,9 @@ parameters:
env(SAML_ROLE_MAPPING): '{}'
env(DATABASE_EMULATE_NATURAL_SORT): 0
######################################################################################################################
# Bulk Info Provider Import Configuration
######################################################################################################################
partdb.bulk_import.batch_size: 20 # Number of parts to process in each batch during bulk operations
partdb.bulk_import.max_parts_per_operation: 1000 # Maximum number of parts allowed per bulk import operation

2
config/services.yaml
View file

@ -17,6 +17,8 @@ services:
bool $gdpr_compliance: '%partdb.gdpr_compliance%'
bool $kernel_debug_enabled: '%kernel.debug%'
string $kernel_cache_dir: '%kernel.cache_dir%'
int $bulkImportBatchSize: '%partdb.bulk_import.batch_size%'
int $bulkImportMaxParts: '%partdb.bulk_import.max_parts_per_operation%'
_instanceof:
App\Services\LabelSystem\PlaceholderProviders\PlaceholderProviderInterface:

84
src/Controller/BulkInfoProviderImportController.php
View file

@ -29,7 +29,7 @@ use App\Entity\Parts\Part;
use App\Entity\Parts\Supplier;
use App\Form\InfoProviderSystem\GlobalFieldMappingType;
use App\Services\InfoProviderSystem\BulkInfoProviderService;
use App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO;
use App\Services\InfoProviderSystem\DTOs\FieldMappingDTO;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
@ -45,10 +45,27 @@ class BulkInfoProviderImportController extends AbstractController
public function __construct(
private readonly BulkInfoProviderService $bulkService,
private readonly EntityManagerInterface $entityManager,
private readonly LoggerInterface $logger
private readonly LoggerInterface $logger,
private readonly int $bulkImportBatchSize,
private readonly int $bulkImportMaxParts
) {
}
/**
* Convert field mappings from array format to FieldMappingDTO[].
*
* @param array $fieldMappings Array of field mapping arrays
* @return FieldMappingDTO[] Array of FieldMappingDTO objects
*/
private function convertFieldMappingsToDto(array $fieldMappings): array
{
$dtos = [];
foreach ($fieldMappings as $mapping) {
$dtos[] = FieldMappingDTO::fromArray($mapping);
}
return $dtos;
}
private function createErrorResponse(string $message, int $statusCode = 400, array $context = []): JsonResponse
{
$this->logger->warning('Bulk import operation failed', array_merge([
@ -122,7 +139,17 @@ class BulkInfoProviderImportController extends AbstractController
return $this->redirectToRoute('homepage');
}
if (count($parts) > 50) {
// Validate against configured maximum
if (count($parts) > $this->bulkImportMaxParts) {
$this->addFlash('error', sprintf(
'Too many parts selected (%d). Maximum allowed is %d parts per operation.',
count($parts),
$this->bulkImportMaxParts
));
return $this->redirectToRoute('homepage');
}
if (count($parts) > ($this->bulkImportMaxParts / 2)) {
$this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.');
}
@ -164,6 +191,13 @@ class BulkInfoProviderImportController extends AbstractController
throw new \RuntimeException('User must be authenticated and of type User');
}
// Validate part count against configuration limit
if (count($parts) > $this->bulkImportMaxParts) {
$this->addFlash('error', "Too many parts selected. Maximum allowed: {$this->bulkImportMaxParts}");
$partIds = array_map(fn($part) => $part->getId(), $parts);
return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
}
// Create and save the job
$job = new BulkInfoProviderImportJob();
$job->setFieldMappings($fieldMappings);
@ -179,13 +213,11 @@ class BulkInfoProviderImportController extends AbstractController
$this->entityManager->flush();
try {
$searchRequest = new BulkSearchRequestDTO(
fieldMappings: $fieldMappings,
prefetchDetails: $prefetchDetails,
parts: $parts
);
$fieldMappingDtos = $this->convertFieldMappingsToDto($fieldMappings);
$searchResultsDto = $this->bulkService->performBulkSearch($parts, $fieldMappingDtos, $prefetchDetails);
$searchResults = $this->bulkService->performBulkSearch($searchRequest);
// Convert DTO back to array format for legacy compatibility
$searchResults = $searchResultsDto->toArray();
// Save search results to job
$job->setSearchResults($job->serializeSearchResults($searchResults));
@ -210,6 +242,7 @@ class BulkInfoProviderImportController extends AbstractController
$this->entityManager->flush();
$this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage());
$partIds = array_map(fn($part) => $part->getId(), $parts);
return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
}
}
@ -441,14 +474,11 @@ class BulkInfoProviderImportController extends AbstractController
$fieldMappings = $job->getFieldMappings();
$prefetchDetails = $job->isPrefetchDetails();
$searchRequest = new BulkSearchRequestDTO(
fieldMappings: $fieldMappings,
prefetchDetails: $prefetchDetails,
parts: [$part]
);
$fieldMappingDtos = $this->convertFieldMappingsToDto($fieldMappings);
try {
$searchResults = $this->bulkService->performBulkSearch($searchRequest);
$searchResultsDto = $this->bulkService->performBulkSearch([$part], $fieldMappingDtos, $prefetchDetails);
$searchResults = $searchResultsDto->toArray();
} catch (\Exception $searchException) {
// Handle "no search results found" as a normal case, not an error
if (str_contains($searchException->getMessage(), 'No search results found')) {
@ -500,13 +530,11 @@ class BulkInfoProviderImportController extends AbstractController
return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]);
}
// Get all part IDs that are not completed or skipped
// Get all parts that are not completed or skipped
$parts = [];
$partIds = [];
foreach ($job->getJobParts() as $jobPart) {
if (!$jobPart->isCompleted() && !$jobPart->isSkipped()) {
$parts[] = $jobPart->getPart();
$partsIds[] = $jobPart->getPart()->getId();
}
}
@ -520,26 +548,22 @@ class BulkInfoProviderImportController extends AbstractController
try {
$fieldMappings = $job->getFieldMappings();
$fieldMappingDtos = $this->convertFieldMappingsToDto($fieldMappings);
$prefetchDetails = $job->isPrefetchDetails();
// Process in batches to reduce memory usage for large operations
$batchSize = 20; // Configurable batch size for memory management
$allResults = [];
$batches = array_chunk($parts, $batchSize);
$batches = array_chunk($parts, $this->bulkImportBatchSize);
foreach ($batches as $batch) {
$searchRequest = new BulkSearchRequestDTO(
fieldMappings: $fieldMappings,
prefetchDetails: $prefetchDetails,
parts: $batch
);
$batchResults = $this->bulkService->performBulkSearch($searchRequest);
$batchResultsDto = $this->bulkService->performBulkSearch($batch, $fieldMappingDtos, $prefetchDetails);
$batchResults = $batchResultsDto->toArray();
$allResults = array_merge($allResults, $batchResults);
// Clear entity manager periodically to prevent memory issues
// Properly manage entity manager memory without losing state
$jobId = $job->getId();
$this->entityManager->clear();
$job = $this->entityManager->find(BulkInfoProviderImportJob::class, $job->getId());
$job = $this->entityManager->find(BulkInfoProviderImportJob::class, $jobId);
}
// Update the job's search results
@ -564,7 +588,7 @@ class BulkInfoProviderImportController extends AbstractController
500,
[
'job_id' => $jobId,
'part_ids' => $partsIds,
'part_count' => count($parts),
'exception' => $e->getMessage()
]
);

244
src/Services/InfoProviderSystem/BulkInfoProviderService.php
View file

@ -4,12 +4,14 @@ declare(strict_types=1);
namespace App\Services\InfoProviderSystem;
use A\B;
use App\Entity\BulkInfoProviderImportJob;
use App\Entity\Base\AbstractPartsContainingDBElement;
use App\Entity\Parts\Part;
use App\Entity\Parts\Supplier;
use App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchResultDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchResponseDTO;
use App\Services\InfoProviderSystem\DTOs\FieldMappingDTO;
use App\Services\InfoProviderSystem\DTOs\PartSearchResultDTO;
use App\Services\InfoProviderSystem\DTOs\SearchResultWithMetadataDTO;
use App\Services\InfoProviderSystem\Providers\BatchInfoProviderInterface;
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
use Doctrine\ORM\EntityManagerInterface;
@ -18,6 +20,9 @@ use Symfony\Component\HttpClient\Exception\ClientException;
final class BulkInfoProviderService
{
/** @var array<string, Supplier|null> Cache for normalized supplier names */
private array $supplierCache = [];
public function __construct(
private readonly PartInfoRetriever $infoRetriever,
private readonly ExistingPartFinder $existingPartFinder,
@ -26,24 +31,31 @@ final class BulkInfoProviderService
private readonly LoggerInterface $logger
) {}
public function performBulkSearch(BulkSearchRequestDTO $request): array
/**
* Perform bulk search across multiple parts and providers.
*
* @param Part[] $parts Array of parts to search for
* @param FieldMappingDTO[] $fieldMappings Array of field mappings defining search strategy
* @param bool $prefetchDetails Whether to prefetch detailed information for results
* @return BulkSearchResponseDTO Structured response containing all search results
* @throws \InvalidArgumentException If no valid parts provided
* @throws \RuntimeException If no search results found for any parts
*/
public function performBulkSearch(array $parts, array $fieldMappings, bool $prefetchDetails = false): BulkSearchResponseDTO
{
$parts = $request->parts;
if (empty($parts)) {
throw new \InvalidArgumentException('No valid parts found for bulk import');
}
$searchResults = [];
$partResults = [];
$hasAnyResults = false;
// Group providers by batch capability
$batchProviders = [];
$regularProviders = [];
foreach ($request->fieldMappings as $mapping) {
$providers = $mapping['providers'] ?? [];
foreach ($providers as $providerKey) {
foreach ($fieldMappings as $mapping) {
foreach ($mapping->providers as $providerKey) {
if (!is_string($providerKey)) {
$this->logger->error('Invalid provider key type', [
'providerKey' => $providerKey,
@ -62,18 +74,14 @@ final class BulkInfoProviderService
}
// Process batch providers first (more efficient)
$batchResults = $this->processBatchProviders($parts, $request->fieldMappings, $batchProviders);
$batchResults = $this->processBatchProviders($parts, $fieldMappings, $batchProviders);
// Process regular providers
$regularResults = $this->processRegularProviders($parts, $request->fieldMappings, $regularProviders, $batchResults);
$regularResults = $this->processRegularProviders($parts, $fieldMappings, $regularProviders, $batchResults);
// Combine and format results
// Combine and format results for each part
foreach ($parts as $part) {
$partResult = [
'part' => $part,
'search_results' => [],
'errors' => []
];
$searchResults = [];
// Get results from batch and regular processing
$allResults = array_merge(
@ -83,24 +91,37 @@ final class BulkInfoProviderService
if (!empty($allResults)) {
$hasAnyResults = true;
$partResult['search_results'] = $this->formatSearchResults($allResults);
$searchResults = $this->formatSearchResults($allResults);
}
$searchResults[] = $partResult;
$partResults[] = new PartSearchResultDTO(
part: $part,
searchResults: $searchResults,
errors: []
);
}
if (!$hasAnyResults) {
throw new \RuntimeException('No search results found for any of the selected parts');
}
return $searchResults;
$response = new BulkSearchResponseDTO($partResults);
// Prefetch details if requested
if ($prefetchDetails) {
$this->prefetchDetailsForResults($response);
}
return $response;
}
/**
* @param Part[] $parts
* @param array $fieldMappings
* @param array<string, BatchInfoProviderInterface> $batchProviders
* @return array<int, BulkSearchResultDTO[]> A list of results indexed by part ID
* Process parts using batch-capable info providers.
*
* @param Part[] $parts Array of parts to search for
* @param FieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param array<string, BatchInfoProviderInterface> $batchProviders Batch providers indexed by key
* @return array<int, BulkSearchResultDTO[]> Results indexed by part ID
*/
private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array
{
@ -119,19 +140,19 @@ final class BulkInfoProviderService
// Map results back to parts
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
if (!in_array($providerKey, $mapping->providers, true)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping['field']);
$keyword = $this->getKeywordFromField($part, $mapping->field);
if ($keyword && isset($providerResults[$keyword])) {
foreach ($providerResults[$keyword] as $dto) {
$batchResults[$part->getId()][] = new BulkSearchResultDTO(
baseDto: $dto,
sourceField: $mapping['field'],
sourceField: $mapping->field,
sourceKeyword: $keyword,
localPart: $this->existingPartFinder->findFirstExisting($dto),
priority: $mapping['priority'] ?? 1
priority: $mapping->priority
);
}
}
@ -149,11 +170,13 @@ final class BulkInfoProviderService
}
/**
* @param Part[] $parts
* @param array $fieldMappings
* @param array<string, InfoProviderInterface> $regularProviders The info providers that do not support batch searching, indexed by their provider key
* @param array $excludeResults
* @return array <int, BulkSearchResultDTO[]> A list of results indexed by part ID
* Process parts using regular (non-batch) info providers.
*
* @param Part[] $parts Array of parts to search for
* @param FieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param array<string, InfoProviderInterface> $regularProviders Regular providers indexed by key
* @param array<int, BulkSearchResultDTO[]> $excludeResults Results to exclude (from batch processing)
* @return array<int, BulkSearchResultDTO[]> Results indexed by part ID
*/
private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array
{
@ -168,14 +191,13 @@ final class BulkInfoProviderService
}
foreach ($fieldMappings as $mapping) {
$field = $mapping['field'];
$providers = array_intersect($mapping['providers'] ?? [], array_keys($regularProviders));
$providers = array_intersect($mapping->providers, array_keys($regularProviders));
if (empty($providers)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $field);
$keyword = $this->getKeywordFromField($part, $mapping->field);
if (!$keyword) {
continue;
}
@ -186,16 +208,16 @@ final class BulkInfoProviderService
foreach ($dtos as $dto) {
$regularResults[$part->getId()][] = new BulkSearchResultDTO(
baseDto: $dto,
sourceField: $field,
sourceField: $mapping->field,
sourceKeyword: $keyword,
localPart: $this->existingPartFinder->findFirstExisting($dto),
priority: $mapping['priority'] ?? 1
priority: $mapping->priority
);
}
} catch (ClientException $e) {
$this->logger->error('Regular search failed', [
'part_id' => $part->getId(),
'field' => $field,
'field' => $mapping->field,
'error' => $e->getMessage()
]);
}
@ -206,10 +228,12 @@ final class BulkInfoProviderService
}
/**
* @param Part[] $parts
* @param array $fieldMappings
* @param string $providerKey
* @return string[]
* Collect unique keywords for a specific provider from all parts and field mappings.
*
* @param Part[] $parts Array of parts to collect keywords from
* @param FieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param string $providerKey The provider key to collect keywords for
* @return string[] Array of unique keywords
*/
private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array
{
@ -217,11 +241,11 @@ final class BulkInfoProviderService
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
if (!in_array($providerKey, $mapping->providers, true)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping['field']);
$keyword = $this->getKeywordFromField($part, $mapping->field);
if ($keyword && !in_array($keyword, $keywords, true)) {
$keywords[] = $keyword;
}
@ -247,22 +271,66 @@ final class BulkInfoProviderService
}
$supplierKey = substr($field, 0, -4);
$supplier = $this->getSupplierByNormalizedName($supplierKey);
if (!$supplier) {
return null;
}
$orderDetail = $part->getOrderdetails()->filter(
fn($od) => $od->getSupplier()?->getId() === $supplier->getId()
)->first();
return $orderDetail !== false ? $orderDetail->getSupplierpartnr() : null;
}
/**
* Get supplier by normalized name with caching to prevent N+1 queries.
*
* @param string $normalizedKey The normalized supplier key to search for
* @return Supplier|null The matching supplier or null if not found
*/
private function getSupplierByNormalizedName(string $normalizedKey): ?Supplier
{
// Check cache first
if (isset($this->supplierCache[$normalizedKey])) {
return $this->supplierCache[$normalizedKey];
}
// Use efficient database query with PHP normalization
// Since DQL doesn't support REPLACE, we'll load all suppliers once and cache the normalization
if (empty($this->supplierCache)) {
$this->loadSuppliersIntoCache();
}
$supplier = $this->supplierCache[$normalizedKey] ?? null;
// Cache the result (including null results to prevent repeated queries)
$this->supplierCache[$normalizedKey] = $supplier;
return $supplier;
}
/**
* Load all suppliers into cache with normalized names to avoid N+1 queries.
*/
private function loadSuppliersIntoCache(): void
{
/** @var Supplier[] $suppliers */
$suppliers = $this->entityManager->getRepository(Supplier::class)->findAll();
foreach ($suppliers as $supplier) {
$normalizedName = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName()));
if ($normalizedName === $supplierKey) {
$orderDetail = $part->getOrderdetails()->filter(
fn($od) => $od->getSupplier()?->getId() === $supplier->getId()
)->first();
return $orderDetail ? $orderDetail->getSupplierpartnr() : null;
}
$this->supplierCache[$normalizedName] = $supplier;
}
return null;
}
/**
* Format and deduplicate search results.
*
* @param BulkSearchResultDTO[] $bulkResults Array of bulk search results
* @return SearchResultWithMetadataDTO[] Array of formatted search results with metadata
*/
private function formatSearchResults(array $bulkResults): array
{
// Sort by priority and remove duplicates
@ -272,38 +340,64 @@ final class BulkInfoProviderService
$seenKeys = [];
foreach ($bulkResults as $result) {
$key = "{$result->provider_key}|{$result->provider_id}";
$key = "{$result->getProviderKey()}|{$result->getProviderId()}";
if (!in_array($key, $seenKeys, true)) {
$seenKeys[] = $key;
$uniqueResults[] = [
'dto' => $result,
'localPart' => $result->localPart,
'source_field' => $result->sourceField,
'source_keyword' => $result->sourceKeyword
];
$uniqueResults[] = new SearchResultWithMetadataDTO(
searchResult: $result,
localPart: $result->localPart,
sourceField: $result->sourceField,
sourceKeyword: $result->sourceKeyword
);
}
}
return $uniqueResults;
}
public function prefetchDetailsForResults(array $searchResults): void
/**
* Prefetch detailed information for search results.
*
* @param BulkSearchResponseDTO|array $searchResults Search results (supports both new DTO and legacy array format)
*/
public function prefetchDetailsForResults($searchResults): void
{
$prefetchCount = 0;
foreach ($searchResults as $partResult) {
foreach ($partResult['search_results'] as $result) {
$dto = $result['dto'];
// Handle both new DTO format and legacy array format for backwards compatibility
if ($searchResults instanceof BulkSearchResponseDTO) {
foreach ($searchResults->partResults as $partResult) {
foreach ($partResult->searchResults as $result) {
$dto = $result->searchResult;
try {
$this->infoRetriever->getDetails($dto->provider_key, $dto->provider_id);
$prefetchCount++;
} catch (\Exception $e) {
$this->logger->warning('Failed to prefetch details for provider part', [
'provider_key' => $dto->provider_key,
'provider_id' => $dto->provider_id,
'error' => $e->getMessage()
]);
try {
$this->infoRetriever->getDetails($dto->getProviderKey(), $dto->getProviderId());
$prefetchCount++;
} catch (\Exception $e) {
$this->logger->warning('Failed to prefetch details for provider part', [
'provider_key' => $dto->getProviderKey(),
'provider_id' => $dto->getProviderId(),
'error' => $e->getMessage()
]);
}
}
}
} else {
// Legacy array format support
foreach ($searchResults as $partResult) {
foreach ($partResult['search_results'] as $result) {
$dto = $result['dto'];
try {
$this->infoRetriever->getDetails($dto->getProviderKey(), $dto->getProviderId());
$prefetchCount++;
} catch (\Exception $e) {
$this->logger->warning('Failed to prefetch details for provider part', [
'provider_key' => $dto->getProviderKey(),
'provider_id' => $dto->getProviderId(),
'error' => $e->getMessage()
]);
}
}
}
}

21
src/Services/InfoProviderSystem/DTOs/BulkSearchRequestDTO.php
View file

@ -1,21 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
readonly class BulkSearchRequestDTO
{
/**
* @param array $fieldMappings
* @param bool $prefetchDetails
* @param Part[] $parts The parts for which the bulk search should be performed.
*/
public function __construct(
public array $fieldMappings,
public bool $prefetchDetails = false,
public array $parts = []
) {}
}

122
src/Services/InfoProviderSystem/DTOs/BulkSearchResponseDTO.php Normal file
View file

@ -0,0 +1,122 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
/**
* Represents the complete response from a bulk info provider search operation.
* This DTO provides type safety and clear structure instead of complex arrays.
*/
readonly class BulkSearchResponseDTO
{
/**
* @param PartSearchResultDTO[] $partResults Array of search results for each part
*/
public function __construct(
public array $partResults
) {}
/**
* Create from legacy array format for backwards compatibility.
* @param array $data Array of part result arrays in legacy format
*/
public static function fromArray(array $data): self
{
$partResults = [];
foreach ($data as $partData) {
$partResults[] = PartSearchResultDTO::fromArray($partData);
}
return new self($partResults);
}
/**
* Convert to legacy array format for backwards compatibility.
*/
public function toArray(): array
{
$result = [];
foreach ($this->partResults as $partResult) {
$result[] = $partResult->toArray();
}
return $result;
}
/**
* Check if any parts have search results.
*/
public function hasAnyResults(): bool
{
foreach ($this->partResults as $partResult) {
if ($partResult->hasResults()) {
return true;
}
}
return false;
}
/**
* Get the total number of search results across all parts.
*/
public function getTotalResultCount(): int
{
$count = 0;
foreach ($this->partResults as $partResult) {
$count += $partResult->getResultCount();
}
return $count;
}
/**
* Get all parts that have search results.
* @return PartSearchResultDTO[]
*/
public function getPartsWithResults(): array
{
return array_filter($this->partResults, fn($result) => $result->hasResults());
}
/**
* Get all parts that have errors.
* @return PartSearchResultDTO[]
*/
public function getPartsWithErrors(): array
{
return array_filter($this->partResults, fn($result) => $result->hasErrors());
}
/**
* Get the number of parts processed.
*/
public function getPartCount(): int
{
return count($this->partResults);
}
/**
* Get the number of parts with successful results.
*/
public function getSuccessfulPartCount(): int
{
return count($this->getPartsWithResults());
}
}

147
src/Services/InfoProviderSystem/DTOs/BulkSearchResultDTO.php
View file

@ -1,32 +1,139 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\ManufacturingStatus;
use App\Entity\Parts\Part;
class BulkSearchResultDTO extends SearchResultDTO
/**
* Represents a search result from bulk search with additional context information.
* Uses composition instead of inheritance for better maintainability.
*/
readonly class BulkSearchResultDTO
{
public function __construct(
SearchResultDTO $baseDto,
public readonly ?string $sourceField = null,
public readonly ?string $sourceKeyword = null,
public readonly ?Part $localPart = null,
public readonly int $priority = 1
) {
parent::__construct(
provider_key: $baseDto->provider_key,
provider_id: $baseDto->provider_id,
name: $baseDto->name,
description: $baseDto->description,
category: $baseDto->category,
manufacturer: $baseDto->manufacturer,
mpn: $baseDto->mpn,
preview_image_url: $baseDto->preview_image_url,
manufacturing_status: $baseDto->manufacturing_status,
provider_url: $baseDto->provider_url,
footprint: $baseDto->footprint
);
/** The base search result DTO containing provider data */
public SearchResultDTO $baseDto,
/** The field that was used to find this result */
public ?string $sourceField = null,
/** The actual keyword that was searched for */
public ?string $sourceKeyword = null,
/** Local part that matches this search result, if any */
public ?Part $localPart = null,
/** Priority for this search result */
public int $priority = 1
) {}
// Delegation methods for SearchResultDTO properties
public function getProviderKey(): string
{
return $this->baseDto->provider_key;
}
public function getProviderId(): string
{
return $this->baseDto->provider_id;
}
public function getName(): string
{
return $this->baseDto->name;
}
public function getDescription(): string
{
return $this->baseDto->description;
}
public function getCategory(): ?string
{
return $this->baseDto->category;
}
public function getManufacturer(): ?string
{
return $this->baseDto->manufacturer;
}
public function getMpn(): ?string
{
return $this->baseDto->mpn;
}
public function getPreviewImageUrl(): ?string
{
return $this->baseDto->preview_image_url;
}
public function getPreviewImageFile(): ?FileDTO
{
return $this->baseDto->preview_image_file;
}
public function getManufacturingStatus(): ?ManufacturingStatus
{
return $this->baseDto->manufacturing_status;
}
public function getProviderUrl(): ?string
{
return $this->baseDto->provider_url;
}
public function getFootprint(): ?string
{
return $this->baseDto->footprint;
}
// Backwards compatibility properties for legacy code
public function __get(string $name): mixed
{
return match ($name) {
'provider_key' => $this->baseDto->provider_key,
'provider_id' => $this->baseDto->provider_id,
'name' => $this->baseDto->name,
'description' => $this->baseDto->description,
'category' => $this->baseDto->category,
'manufacturer' => $this->baseDto->manufacturer,
'mpn' => $this->baseDto->mpn,
'preview_image_url' => $this->baseDto->preview_image_url,
'preview_image_file' => $this->baseDto->preview_image_file,
'manufacturing_status' => $this->baseDto->manufacturing_status,
'provider_url' => $this->baseDto->provider_url,
'footprint' => $this->baseDto->footprint,
default => throw new \InvalidArgumentException("Property '{$name}' does not exist")
};
}
/**
* Magic isset method for backwards compatibility.
*/
public function __isset(string $name): bool
{
return in_array($name, [
'provider_key', 'provider_id', 'name', 'description', 'category',
'manufacturer', 'mpn', 'preview_image_url', 'preview_image_file',
'manufacturing_status', 'provider_url', 'footprint'
], true);
}
}

92
src/Services/InfoProviderSystem/DTOs/FieldMappingDTO.php Normal file
View file

@ -0,0 +1,92 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
/**
* Represents a mapping between a part field and the info providers that should search in that field.
* This DTO provides type safety and better structure than raw arrays for field mapping configuration.
*/
readonly class FieldMappingDTO
{
/**
* @param string $field The field to search in (e.g., 'mpn', 'name', or supplier-specific fields like 'digikey_spn')
* @param string[] $providers Array of provider keys to search with (e.g., ['digikey', 'farnell'])
* @param int $priority Priority for this field mapping (1-10, lower numbers = higher priority)
*/
public function __construct(
public string $field,
public array $providers,
public int $priority = 1
) {
if ($priority < 1 || $priority > 10) {
throw new \InvalidArgumentException('Priority must be between 1 and 10');
}
}
/**
* Create a FieldMappingDTO from legacy array format.
* @param array{field: string, providers: string[], priority?: int} $data
*/
public static function fromArray(array $data): self
{
return new self(
field: $data['field'],
providers: $data['providers'] ?? [],
priority: $data['priority'] ?? 1
);
}
/**
* Convert this DTO to the legacy array format for backwards compatibility.
* @return array{field: string, providers: string[], priority: int}
*/
public function toArray(): array
{
return [
'field' => $this->field,
'providers' => $this->providers,
'priority' => $this->priority,
];
}
/**
* Check if this field mapping is for a supplier part number field.
*/
public function isSupplierPartNumberField(): bool
{
return str_ends_with($this->field, '_spn');
}
/**
* Get the supplier key from a supplier part number field.
* Returns null if this is not a supplier part number field.
*/
public function getSupplierKey(): ?string
{
if (!$this->isSupplierPartNumberField()) {
return null;
}
return substr($this->field, 0, -4);
}
}

114
src/Services/InfoProviderSystem/DTOs/PartSearchResultDTO.php Normal file
View file

@ -0,0 +1,114 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
/**
* Represents the search results for a single part from bulk info provider search.
* This DTO provides type safety and clear structure for part search results.
*/
readonly class PartSearchResultDTO
{
/**
* @param Part $part The part that was searched for
* @param SearchResultWithMetadataDTO[] $searchResults Array of search results found for this part
* @param string[] $errors Array of error messages encountered during search
*/
public function __construct(
public Part $part,
public array $searchResults = [],
public array $errors = []
) {}
/**
* Create from legacy array format for backwards compatibility.
* @param array{part: Part, search_results: array, errors: string[]} $data
*/
public static function fromArray(array $data): self
{
$searchResults = [];
foreach ($data['search_results'] as $result) {
$searchResults[] = SearchResultWithMetadataDTO::fromArray($result);
}
return new self(
part: $data['part'],
searchResults: $searchResults,
errors: $data['errors'] ?? []
);
}
/**
* Convert to legacy array format for backwards compatibility.
* @return array{part: Part, search_results: array, errors: string[]}
*/
public function toArray(): array
{
$searchResults = [];
foreach ($this->searchResults as $result) {
$searchResults[] = $result->toArray();
}
return [
'part' => $this->part,
'search_results' => $searchResults,
'errors' => $this->errors,
];
}
/**
* Check if this part has any search results.
*/
public function hasResults(): bool
{
return !empty($this->searchResults);
}
/**
* Check if this part has any errors.
*/
public function hasErrors(): bool
{
return !empty($this->errors);
}
/**
* Get the number of search results for this part.
*/
public function getResultCount(): int
{
return count($this->searchResults);
}
/**
* Get search results sorted by priority (ascending).
* @return SearchResultWithMetadataDTO[]
*/
public function getResultsSortedByPriority(): array
{
$results = $this->searchResults;
usort($results, fn($a, $b) => $a->getPriority() <=> $b->getPriority());
return $results;
}
}

95
src/Services/InfoProviderSystem/DTOs/SearchResultWithMetadataDTO.php Normal file
View file

@ -0,0 +1,95 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
/**
* Represents a search result with additional metadata about how it was found.
* This DTO encapsulates both the search result data and the context of the search.
*/
readonly class SearchResultWithMetadataDTO
{
public function __construct(
/** The search result DTO containing part information from the provider */
public BulkSearchResultDTO $searchResult,
/** Local part that matches this search result, if any */
public ?Part $localPart = null,
/** The field that was used to find this result (e.g., 'mpn', 'name') */
public ?string $sourceField = null,
/** The actual keyword/value that was searched for */
public ?string $sourceKeyword = null
) {}
/**
* Create from legacy array format for backwards compatibility.
* @param array{dto: BulkSearchResultDTO, localPart: ?Part, source_field: string, source_keyword: string} $data
*/
public static function fromArray(array $data): self
{
return new self(
searchResult: $data['dto'],
localPart: $data['localPart'] ?? null,
sourceField: $data['source_field'] ?? null,
sourceKeyword: $data['source_keyword'] ?? null
);
}
/**
* Convert to legacy array format for backwards compatibility.
* @return array{dto: BulkSearchResultDTO, localPart: ?Part, source_field: ?string, source_keyword: ?string}
*/
public function toArray(): array
{
return [
'dto' => $this->searchResult,
'localPart' => $this->localPart,
'source_field' => $this->sourceField,
'source_keyword' => $this->sourceKeyword,
];
}
/**
* Get the priority of this search result.
*/
public function getPriority(): int
{
return $this->searchResult->priority;
}
/**
* Get the provider key from the search result.
*/
public function getProviderKey(): string
{
return $this->searchResult->getProviderKey();
}
/**
* Get the provider ID from the search result.
*/
public function getProviderId(): string
{
return $this->searchResult->getProviderId();
}
}

19
src/Services/InfoProviderSystem/Providers/MouserProvider.php
View file

@ -132,6 +132,15 @@ class MouserProvider implements InfoProviderInterface
],
]);
// Check for API errors before processing response
if ($response->getStatusCode() !== 200) {
throw new \RuntimeException(sprintf(
'Mouser API returned HTTP %d: %s',
$response->getStatusCode(),
$response->getContent(false)
));
}
return $this->responseToDTOArray($response);
}
@ -169,6 +178,16 @@ class MouserProvider implements InfoProviderInterface
]
],
]);
// Check for API errors before processing response
if ($response->getStatusCode() !== 200) {
throw new \RuntimeException(sprintf(
'Mouser API returned HTTP %d: %s',
$response->getStatusCode(),
$response->getContent(false)
));
}
$tmp = $this->responseToDTOArray($response);
//Ensure that we have exactly one result

16
templates/info_providers/bulk_import/step1.html.twig
View file

@ -140,15 +140,14 @@
data-controller="field-mapping"
data-field-mapping-mapping-index-value="{{ form.field_mappings|length }}"
data-field-mapping-max-mappings-value="{{ fieldChoices|length }}"
data-field-mapping-prototype-value="{{ form_widget(form.field_mappings.vars.prototype)|e('js') }}"
data-field-mapping-prototype-value="{{ form_widget(form.field_mappings.vars.prototype)|e('html_attr') }}"
data-field-mapping-max-mappings-reached-message-value="{{ 'info_providers.bulk_import.max_mappings_reached'|trans|e('js') }}">
<div class="card-header">
<h5 class="mb-0">{% trans %}info_providers.bulk_import.field_mappings{% endtrans %}</h5>
<small class="text-muted">{% trans %}info_providers.bulk_import.field_mappings_help{% endtrans %}</small>
</div>
<div class="card-body">
<div class="table-responsive">
<table class="table table-striped">
<table class="table table-striped">
<thead>
<tr>
<th>{% trans %}info_providers.bulk_search.search_field{% endtrans %}</th>
@ -171,11 +170,12 @@
</tr>
{% endfor %}
</tbody>
</table>
<button type="button" class="btn btn-success btn-sm" id="addMappingBtn" data-field-mapping-target="addButton">
<i class="fas fa-plus"></i> {% trans %}info_providers.bulk_import.add_mapping{% endtrans %}
</button>
</div>
</table>
<button type="button" class="btn btn-success btn-sm" id="addMappingBtn"
data-field-mapping-target="addButton"
data-action="click->field-mapping#addMapping">
<i class="fas fa-plus"></i> {% trans %}info_providers.bulk_import.add_mapping{% endtrans %}
</button>
</div>
</div>

48
tests/Controller/BulkInfoProviderImportControllerTest.php
View file

@ -657,20 +657,16 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
// Test that the service can extract keywords from parts
$bulkService = $client->getContainer()->get(\App\Services\InfoProviderSystem\BulkInfoProviderService::class);
// Create a test request to verify the service works
$request = new \App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO(
fieldMappings: [
['field' => 'name', 'providers' => ['test'], 'priority' => 1],
['field' => 'mpn', 'providers' => ['test'], 'priority' => 2]
],
prefetchDetails: false,
parts: [$part]
);
// Create field mappings to verify the service works
$fieldMappings = [
new \App\Services\InfoProviderSystem\DTOs\FieldMappingDTO('name', ['test'], 1),
new \App\Services\InfoProviderSystem\DTOs\FieldMappingDTO('mpn', ['test'], 2)
];
// The service may return an empty result or throw when no results are found
try {
$result = $bulkService->performBulkSearch($request);
$this->assertIsArray($result);
$result = $bulkService->performBulkSearch([$part], $fieldMappings, false);
$this->assertInstanceOf(\App\Services\InfoProviderSystem\DTOs\BulkSearchResponseDTO::class, $result);
} catch (\RuntimeException $e) {
$this->assertStringContainsString('No search results found', $e->getMessage());
}
@ -792,19 +788,15 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
// Test that the service can handle supplier part number fields
$bulkService = $client->getContainer()->get(\App\Services\InfoProviderSystem\BulkInfoProviderService::class);
// Create a test request with supplier SPN field mapping
$request = new \App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO(
fieldMappings: [
['field' => 'invalid_field', 'providers' => ['test'], 'priority' => 1],
['field' => 'test_supplier_spn', 'providers' => ['test'], 'priority' => 2]
],
prefetchDetails: false,
parts: [$part]
);
// Create field mappings with supplier SPN field mapping
$fieldMappings = [
new \App\Services\InfoProviderSystem\DTOs\FieldMappingDTO('invalid_field', ['test'], 1),
new \App\Services\InfoProviderSystem\DTOs\FieldMappingDTO('test_supplier_spn', ['test'], 2)
];
// The service should be able to process the request and throw an exception when no results are found
try {
$bulkService->performBulkSearch($request);
$bulkService->performBulkSearch([$part], $fieldMappings, false);
$this->fail('Expected RuntimeException to be thrown when no search results are found');
} catch (\RuntimeException $e) {
$this->assertStringContainsString('No search results found', $e->getMessage());
@ -827,18 +819,14 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
// Test that the service can handle batch processing
$bulkService = $client->getContainer()->get(\App\Services\InfoProviderSystem\BulkInfoProviderService::class);
// Create a test request with multiple keywords
$request = new \App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO(
fieldMappings: [
['field' => 'name', 'providers' => ['lcsc'], 'priority' => 1]
],
prefetchDetails: false,
parts: [$part]
);
// Create field mappings with multiple keywords
$fieldMappings = [
new \App\Services\InfoProviderSystem\DTOs\FieldMappingDTO('name', ['lcsc'], 1)
];
// The service should be able to process the request and throw an exception when no results are found
try {
$bulkService->performBulkSearch($request);
$bulkService->performBulkSearch([$part], $fieldMappings, false);
$this->fail('Expected RuntimeException to be thrown when no search results are found');
} catch (\RuntimeException $e) {
$this->assertStringContainsString('No search results found', $e->getMessage());