Pass parts object directly to BulkSearchRequestDTO and added some syntax hints

This commit is contained in:
Jan Böhmer 2025-09-14 22:56:12 +02:00
parent 0e99faee0a
commit 41a7238ab7
5 changed files with 60 additions and 40 deletions

View file

@ -182,7 +182,7 @@ class BulkInfoProviderImportController extends AbstractController
$searchRequest = new BulkSearchRequestDTO( $searchRequest = new BulkSearchRequestDTO(
fieldMappings: $fieldMappings, fieldMappings: $fieldMappings,
prefetchDetails: $prefetchDetails, prefetchDetails: $prefetchDetails,
partIds: $partIds parts: $parts
); );
$searchResults = $this->bulkService->performBulkSearch($searchRequest); $searchResults = $this->bulkService->performBulkSearch($searchRequest);
@ -444,7 +444,7 @@ class BulkInfoProviderImportController extends AbstractController
$searchRequest = new BulkSearchRequestDTO( $searchRequest = new BulkSearchRequestDTO(
fieldMappings: $fieldMappings, fieldMappings: $fieldMappings,
prefetchDetails: $prefetchDetails, prefetchDetails: $prefetchDetails,
partIds: [$partId] parts: [$part]
); );
try { try {
@ -501,14 +501,16 @@ class BulkInfoProviderImportController extends AbstractController
} }
// Get all part IDs that are not completed or skipped // Get all part IDs that are not completed or skipped
$parts = [];
$partIds = []; $partIds = [];
foreach ($job->getJobParts() as $jobPart) { foreach ($job->getJobParts() as $jobPart) {
if (!$jobPart->isCompleted() && !$jobPart->isSkipped()) { if (!$jobPart->isCompleted() && !$jobPart->isSkipped()) {
$partIds[] = $jobPart->getPart()->getId(); $parts[] = $jobPart->getPart();
$partsIds[] = $jobPart->getPart()->getId();
} }
} }
if (empty($partIds)) { if (empty($parts)) {
return $this->json([ return $this->json([
'success' => true, 'success' => true,
'message' => 'No parts to research', 'message' => 'No parts to research',
@ -523,13 +525,13 @@ class BulkInfoProviderImportController extends AbstractController
// Process in batches to reduce memory usage for large operations // Process in batches to reduce memory usage for large operations
$batchSize = 20; // Configurable batch size for memory management $batchSize = 20; // Configurable batch size for memory management
$allResults = []; $allResults = [];
$batches = array_chunk($partIds, $batchSize); $batches = array_chunk($parts, $batchSize);
foreach ($batches as $batch) { foreach ($batches as $batch) {
$searchRequest = new BulkSearchRequestDTO( $searchRequest = new BulkSearchRequestDTO(
fieldMappings: $fieldMappings, fieldMappings: $fieldMappings,
prefetchDetails: $prefetchDetails, prefetchDetails: $prefetchDetails,
partIds: $batch parts: $batch
); );
$batchResults = $this->bulkService->performBulkSearch($searchRequest); $batchResults = $this->bulkService->performBulkSearch($searchRequest);
@ -552,8 +554,8 @@ class BulkInfoProviderImportController extends AbstractController
return $this->json([ return $this->json([
'success' => true, 'success' => true,
'researched_count' => count($partIds), 'researched_count' => count($parts),
'message' => sprintf('Successfully researched %d parts', count($partIds)) 'message' => sprintf('Successfully researched %d parts', count($parts))
]); ]);
} catch (\Exception $e) { } catch (\Exception $e) {
@ -562,7 +564,7 @@ class BulkInfoProviderImportController extends AbstractController
500, 500,
[ [
'job_id' => $jobId, 'job_id' => $jobId,
'part_ids' => $partIds, 'part_ids' => $partsIds,
'exception' => $e->getMessage() 'exception' => $e->getMessage()
] ]
); );

View file

@ -397,6 +397,10 @@ class BulkInfoProviderImportJob extends AbstractDBElement
return $completed >= $total; return $completed >= $total;
} }
/**
* @param array $searchResults
* @return array{part_id: int, search_results: array{dto: array{provider_key: string, provider_id: string, name: string, description: string, manufacturer: string, mpn: string, provider_url: string, preview_image_url: string, _source_field: string|null, _source_keyword: string|null}, localPart: int|null}[], errors: string[]}[]
*/
public function serializeSearchResults(array $searchResults): array public function serializeSearchResults(array $searchResults): array
{ {
$serialized = []; $serialized = [];
@ -433,6 +437,10 @@ class BulkInfoProviderImportJob extends AbstractDBElement
return $serialized; return $serialized;
} }
/**
* @param EntityManagerInterface|null $entityManager
* @return array{part: Part, search_results: array{dto: SearchResultDTO, localPart: Part|null, source_field: string|null, source_keyword: string|null}[], errors: string[]}[]
*/
public function deserializeSearchResults(?EntityManagerInterface $entityManager = null): array public function deserializeSearchResults(?EntityManagerInterface $entityManager = null): array
{ {
if (empty($this->searchResults)) { if (empty($this->searchResults)) {

View file

@ -27,11 +27,7 @@ final class BulkInfoProviderService
public function performBulkSearch(BulkSearchRequestDTO $request): array public function performBulkSearch(BulkSearchRequestDTO $request): array
{ {
// Convert string IDs to integers $parts = $request->parts;
$partIds = array_map('intval', $request->partIds);
$partRepository = $this->entityManager->getRepository(Part::class);
$parts = $partRepository->getElementsFromIDArray($partIds);
if (empty($parts)) { if (empty($parts)) {
throw new \InvalidArgumentException('No valid parts found for bulk import'); throw new \InvalidArgumentException('No valid parts found for bulk import');
@ -43,7 +39,7 @@ final class BulkInfoProviderService
// Group providers by batch capability // Group providers by batch capability
$batchProviders = []; $batchProviders = [];
$regularProviders = []; $regularProviders = [];
foreach ($request->fieldMappings as $mapping) { foreach ($request->fieldMappings as $mapping) {
$providers = $mapping['providers'] ?? []; $providers = $mapping['providers'] ?? [];
foreach ($providers as $providerKey) { foreach ($providers as $providerKey) {
@ -54,7 +50,7 @@ final class BulkInfoProviderService
]); ]);
continue; continue;
} }
$provider = $this->providerRegistry->getProviderByKey($providerKey); $provider = $this->providerRegistry->getProviderByKey($providerKey);
if ($provider instanceof BatchInfoProviderInterface) { if ($provider instanceof BatchInfoProviderInterface) {
$batchProviders[$providerKey] = $provider; $batchProviders[$providerKey] = $provider;
@ -66,7 +62,7 @@ final class BulkInfoProviderService
// Process batch providers first (more efficient) // Process batch providers first (more efficient)
$batchResults = $this->processBatchProviders($parts, $request->fieldMappings, $batchProviders); $batchResults = $this->processBatchProviders($parts, $request->fieldMappings, $batchProviders);
// Process regular providers // Process regular providers
$regularResults = $this->processRegularProviders($parts, $request->fieldMappings, $regularProviders, $batchResults); $regularResults = $this->processRegularProviders($parts, $request->fieldMappings, $regularProviders, $batchResults);
@ -102,24 +98,24 @@ final class BulkInfoProviderService
private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array
{ {
$batchResults = []; $batchResults = [];
foreach ($batchProviders as $providerKey => $provider) { foreach ($batchProviders as $providerKey => $provider) {
$keywords = $this->collectKeywordsForProvider($parts, $fieldMappings, $providerKey); $keywords = $this->collectKeywordsForProvider($parts, $fieldMappings, $providerKey);
if (empty($keywords)) { if (empty($keywords)) {
continue; continue;
} }
try { try {
$providerResults = $provider->searchByKeywordsBatch($keywords); $providerResults = $provider->searchByKeywordsBatch($keywords);
// Map results back to parts // Map results back to parts
foreach ($parts as $part) { foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) { foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) { if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
continue; continue;
} }
$keyword = $this->getKeywordFromField($part, $mapping['field']); $keyword = $this->getKeywordFromField($part, $mapping['field']);
if ($keyword && isset($providerResults[$keyword])) { if ($keyword && isset($providerResults[$keyword])) {
foreach ($providerResults[$keyword] as $dto) { foreach ($providerResults[$keyword] as $dto) {
@ -145,13 +141,20 @@ final class BulkInfoProviderService
return $batchResults; return $batchResults;
} }
/**
* @param Part[] $parts
* @param array $fieldMappings
* @param array $regularProviders
* @param array $excludeResults
* @return array
*/
private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array
{ {
$regularResults = []; $regularResults = [];
foreach ($parts as $part) { foreach ($parts as $part) {
$regularResults[$part->getId()] = []; $regularResults[$part->getId()] = [];
// Skip if we already have batch results for this part // Skip if we already have batch results for this part
if (!empty($excludeResults[$part->getId()] ?? [])) { if (!empty($excludeResults[$part->getId()] ?? [])) {
continue; continue;
@ -160,7 +163,7 @@ final class BulkInfoProviderService
foreach ($fieldMappings as $mapping) { foreach ($fieldMappings as $mapping) {
$field = $mapping['field']; $field = $mapping['field'];
$providers = array_intersect($mapping['providers'] ?? [], array_keys($regularProviders)); $providers = array_intersect($mapping['providers'] ?? [], array_keys($regularProviders));
if (empty($providers)) { if (empty($providers)) {
continue; continue;
} }
@ -172,7 +175,7 @@ final class BulkInfoProviderService
try { try {
$dtos = $this->infoRetriever->searchByKeyword($keyword, $providers); $dtos = $this->infoRetriever->searchByKeyword($keyword, $providers);
foreach ($dtos as $dto) { foreach ($dtos as $dto) {
$regularResults[$part->getId()][] = new BulkSearchResultDTO( $regularResults[$part->getId()][] = new BulkSearchResultDTO(
baseDto: $dto, baseDto: $dto,
@ -198,13 +201,13 @@ final class BulkInfoProviderService
private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array
{ {
$keywords = []; $keywords = [];
foreach ($parts as $part) { foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) { foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) { if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
continue; continue;
} }
$keyword = $this->getKeywordFromField($part, $mapping['field']); $keyword = $this->getKeywordFromField($part, $mapping['field']);
if ($keyword && !in_array($keyword, $keywords, true)) { if ($keyword && !in_array($keyword, $keywords, true)) {
$keywords[] = $keyword; $keywords[] = $keyword;
@ -251,10 +254,10 @@ final class BulkInfoProviderService
{ {
// Sort by priority and remove duplicates // Sort by priority and remove duplicates
usort($bulkResults, fn($a, $b) => $a->priority <=> $b->priority); usort($bulkResults, fn($a, $b) => $a->priority <=> $b->priority);
$uniqueResults = []; $uniqueResults = [];
$seenKeys = []; $seenKeys = [];
foreach ($bulkResults as $result) { foreach ($bulkResults as $result) {
$key = "{$result->provider_key}|{$result->provider_id}"; $key = "{$result->provider_key}|{$result->provider_id}";
if (!in_array($key, $seenKeys, true)) { if (!in_array($key, $seenKeys, true)) {
@ -294,4 +297,4 @@ final class BulkInfoProviderService
$this->logger->info("Prefetched details for {$prefetchCount} search results"); $this->logger->info("Prefetched details for {$prefetchCount} search results");
} }
} }

View file

@ -4,11 +4,18 @@ declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs; namespace App\Services\InfoProviderSystem\DTOs;
class BulkSearchRequestDTO use App\Entity\Parts\Part;
readonly class BulkSearchRequestDTO
{ {
/**
* @param array $fieldMappings
* @param bool $prefetchDetails
* @param Part[] $parts The parts for which the bulk search should be performed.
*/
public function __construct( public function __construct(
public readonly array $fieldMappings, public array $fieldMappings,
public readonly bool $prefetchDetails = false, public bool $prefetchDetails = false,
public readonly array $partIds = [] public array $parts = []
) {} ) {}
} }

View file

@ -664,7 +664,7 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
['field' => 'mpn', 'providers' => ['test'], 'priority' => 2] ['field' => 'mpn', 'providers' => ['test'], 'priority' => 2]
], ],
prefetchDetails: false, prefetchDetails: false,
partIds: [$part->getId()] parts: [$part->getId()]
); );
// The service may return an empty result or throw when no results are found // The service may return an empty result or throw when no results are found
@ -766,7 +766,7 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
$this->assertResponseStatusCodeSame(Response::HTTP_OK); $this->assertResponseStatusCodeSame(Response::HTTP_OK);
// Find job from database to avoid detached entity errors // Find job from database to avoid detached entity errors
$jobId = $job->getId(); $jobId = $job->getId();
$entityManager->clear(); $entityManager->clear();
$persistedJob = $entityManager->find(BulkInfoProviderImportJob::class, $jobId); $persistedJob = $entityManager->find(BulkInfoProviderImportJob::class, $jobId);
@ -799,7 +799,7 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
['field' => 'test_supplier_spn', 'providers' => ['test'], 'priority' => 2] ['field' => 'test_supplier_spn', 'providers' => ['test'], 'priority' => 2]
], ],
prefetchDetails: false, prefetchDetails: false,
partIds: [$part->getId()] parts: [$part->getId()]
); );
// The service should be able to process the request and throw an exception when no results are found // The service should be able to process the request and throw an exception when no results are found
@ -833,7 +833,7 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
['field' => 'name', 'providers' => ['lcsc'], 'priority' => 1] ['field' => 'name', 'providers' => ['lcsc'], 'priority' => 1]
], ],
prefetchDetails: false, prefetchDetails: false,
partIds: [$part->getId()] parts: [$part->getId()]
); );
// The service should be able to process the request and throw an exception when no results are found // The service should be able to process the request and throw an exception when no results are found
@ -962,4 +962,4 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
$entityManager->remove($job); $entityManager->remove($job);
$entityManager->flush(); $entityManager->flush();
} }
} }