mirror of
https://github.com/Part-DB/Part-DB-server.git
synced 2025-12-06 11:09:29 +00:00
Pass parts object directly to BulkSearchRequestDTO and added some syntax hints
This commit is contained in:
parent
0e99faee0a
commit
41a7238ab7
5 changed files with 60 additions and 40 deletions
|
|
@ -182,7 +182,7 @@ class BulkInfoProviderImportController extends AbstractController
|
||||||
$searchRequest = new BulkSearchRequestDTO(
|
$searchRequest = new BulkSearchRequestDTO(
|
||||||
fieldMappings: $fieldMappings,
|
fieldMappings: $fieldMappings,
|
||||||
prefetchDetails: $prefetchDetails,
|
prefetchDetails: $prefetchDetails,
|
||||||
partIds: $partIds
|
parts: $parts
|
||||||
);
|
);
|
||||||
|
|
||||||
$searchResults = $this->bulkService->performBulkSearch($searchRequest);
|
$searchResults = $this->bulkService->performBulkSearch($searchRequest);
|
||||||
|
|
@ -444,7 +444,7 @@ class BulkInfoProviderImportController extends AbstractController
|
||||||
$searchRequest = new BulkSearchRequestDTO(
|
$searchRequest = new BulkSearchRequestDTO(
|
||||||
fieldMappings: $fieldMappings,
|
fieldMappings: $fieldMappings,
|
||||||
prefetchDetails: $prefetchDetails,
|
prefetchDetails: $prefetchDetails,
|
||||||
partIds: [$partId]
|
parts: [$part]
|
||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -501,14 +501,16 @@ class BulkInfoProviderImportController extends AbstractController
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get all part IDs that are not completed or skipped
|
// Get all part IDs that are not completed or skipped
|
||||||
|
$parts = [];
|
||||||
$partIds = [];
|
$partIds = [];
|
||||||
foreach ($job->getJobParts() as $jobPart) {
|
foreach ($job->getJobParts() as $jobPart) {
|
||||||
if (!$jobPart->isCompleted() && !$jobPart->isSkipped()) {
|
if (!$jobPart->isCompleted() && !$jobPart->isSkipped()) {
|
||||||
$partIds[] = $jobPart->getPart()->getId();
|
$parts[] = $jobPart->getPart();
|
||||||
|
$partsIds[] = $jobPart->getPart()->getId();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (empty($partIds)) {
|
if (empty($parts)) {
|
||||||
return $this->json([
|
return $this->json([
|
||||||
'success' => true,
|
'success' => true,
|
||||||
'message' => 'No parts to research',
|
'message' => 'No parts to research',
|
||||||
|
|
@ -523,13 +525,13 @@ class BulkInfoProviderImportController extends AbstractController
|
||||||
// Process in batches to reduce memory usage for large operations
|
// Process in batches to reduce memory usage for large operations
|
||||||
$batchSize = 20; // Configurable batch size for memory management
|
$batchSize = 20; // Configurable batch size for memory management
|
||||||
$allResults = [];
|
$allResults = [];
|
||||||
$batches = array_chunk($partIds, $batchSize);
|
$batches = array_chunk($parts, $batchSize);
|
||||||
|
|
||||||
foreach ($batches as $batch) {
|
foreach ($batches as $batch) {
|
||||||
$searchRequest = new BulkSearchRequestDTO(
|
$searchRequest = new BulkSearchRequestDTO(
|
||||||
fieldMappings: $fieldMappings,
|
fieldMappings: $fieldMappings,
|
||||||
prefetchDetails: $prefetchDetails,
|
prefetchDetails: $prefetchDetails,
|
||||||
partIds: $batch
|
parts: $batch
|
||||||
);
|
);
|
||||||
|
|
||||||
$batchResults = $this->bulkService->performBulkSearch($searchRequest);
|
$batchResults = $this->bulkService->performBulkSearch($searchRequest);
|
||||||
|
|
@ -552,8 +554,8 @@ class BulkInfoProviderImportController extends AbstractController
|
||||||
|
|
||||||
return $this->json([
|
return $this->json([
|
||||||
'success' => true,
|
'success' => true,
|
||||||
'researched_count' => count($partIds),
|
'researched_count' => count($parts),
|
||||||
'message' => sprintf('Successfully researched %d parts', count($partIds))
|
'message' => sprintf('Successfully researched %d parts', count($parts))
|
||||||
]);
|
]);
|
||||||
|
|
||||||
} catch (\Exception $e) {
|
} catch (\Exception $e) {
|
||||||
|
|
@ -562,7 +564,7 @@ class BulkInfoProviderImportController extends AbstractController
|
||||||
500,
|
500,
|
||||||
[
|
[
|
||||||
'job_id' => $jobId,
|
'job_id' => $jobId,
|
||||||
'part_ids' => $partIds,
|
'part_ids' => $partsIds,
|
||||||
'exception' => $e->getMessage()
|
'exception' => $e->getMessage()
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -397,6 +397,10 @@ class BulkInfoProviderImportJob extends AbstractDBElement
|
||||||
return $completed >= $total;
|
return $completed >= $total;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param array $searchResults
|
||||||
|
* @return array{part_id: int, search_results: array{dto: array{provider_key: string, provider_id: string, name: string, description: string, manufacturer: string, mpn: string, provider_url: string, preview_image_url: string, _source_field: string|null, _source_keyword: string|null}, localPart: int|null}[], errors: string[]}[]
|
||||||
|
*/
|
||||||
public function serializeSearchResults(array $searchResults): array
|
public function serializeSearchResults(array $searchResults): array
|
||||||
{
|
{
|
||||||
$serialized = [];
|
$serialized = [];
|
||||||
|
|
@ -433,6 +437,10 @@ class BulkInfoProviderImportJob extends AbstractDBElement
|
||||||
return $serialized;
|
return $serialized;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param EntityManagerInterface|null $entityManager
|
||||||
|
* @return array{part: Part, search_results: array{dto: SearchResultDTO, localPart: Part|null, source_field: string|null, source_keyword: string|null}[], errors: string[]}[]
|
||||||
|
*/
|
||||||
public function deserializeSearchResults(?EntityManagerInterface $entityManager = null): array
|
public function deserializeSearchResults(?EntityManagerInterface $entityManager = null): array
|
||||||
{
|
{
|
||||||
if (empty($this->searchResults)) {
|
if (empty($this->searchResults)) {
|
||||||
|
|
|
||||||
|
|
@ -27,11 +27,7 @@ final class BulkInfoProviderService
|
||||||
|
|
||||||
public function performBulkSearch(BulkSearchRequestDTO $request): array
|
public function performBulkSearch(BulkSearchRequestDTO $request): array
|
||||||
{
|
{
|
||||||
// Convert string IDs to integers
|
$parts = $request->parts;
|
||||||
$partIds = array_map('intval', $request->partIds);
|
|
||||||
|
|
||||||
$partRepository = $this->entityManager->getRepository(Part::class);
|
|
||||||
$parts = $partRepository->getElementsFromIDArray($partIds);
|
|
||||||
|
|
||||||
if (empty($parts)) {
|
if (empty($parts)) {
|
||||||
throw new \InvalidArgumentException('No valid parts found for bulk import');
|
throw new \InvalidArgumentException('No valid parts found for bulk import');
|
||||||
|
|
@ -43,7 +39,7 @@ final class BulkInfoProviderService
|
||||||
// Group providers by batch capability
|
// Group providers by batch capability
|
||||||
$batchProviders = [];
|
$batchProviders = [];
|
||||||
$regularProviders = [];
|
$regularProviders = [];
|
||||||
|
|
||||||
foreach ($request->fieldMappings as $mapping) {
|
foreach ($request->fieldMappings as $mapping) {
|
||||||
$providers = $mapping['providers'] ?? [];
|
$providers = $mapping['providers'] ?? [];
|
||||||
foreach ($providers as $providerKey) {
|
foreach ($providers as $providerKey) {
|
||||||
|
|
@ -54,7 +50,7 @@ final class BulkInfoProviderService
|
||||||
]);
|
]);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
$provider = $this->providerRegistry->getProviderByKey($providerKey);
|
$provider = $this->providerRegistry->getProviderByKey($providerKey);
|
||||||
if ($provider instanceof BatchInfoProviderInterface) {
|
if ($provider instanceof BatchInfoProviderInterface) {
|
||||||
$batchProviders[$providerKey] = $provider;
|
$batchProviders[$providerKey] = $provider;
|
||||||
|
|
@ -66,7 +62,7 @@ final class BulkInfoProviderService
|
||||||
|
|
||||||
// Process batch providers first (more efficient)
|
// Process batch providers first (more efficient)
|
||||||
$batchResults = $this->processBatchProviders($parts, $request->fieldMappings, $batchProviders);
|
$batchResults = $this->processBatchProviders($parts, $request->fieldMappings, $batchProviders);
|
||||||
|
|
||||||
// Process regular providers
|
// Process regular providers
|
||||||
$regularResults = $this->processRegularProviders($parts, $request->fieldMappings, $regularProviders, $batchResults);
|
$regularResults = $this->processRegularProviders($parts, $request->fieldMappings, $regularProviders, $batchResults);
|
||||||
|
|
||||||
|
|
@ -102,24 +98,24 @@ final class BulkInfoProviderService
|
||||||
private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array
|
private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array
|
||||||
{
|
{
|
||||||
$batchResults = [];
|
$batchResults = [];
|
||||||
|
|
||||||
foreach ($batchProviders as $providerKey => $provider) {
|
foreach ($batchProviders as $providerKey => $provider) {
|
||||||
$keywords = $this->collectKeywordsForProvider($parts, $fieldMappings, $providerKey);
|
$keywords = $this->collectKeywordsForProvider($parts, $fieldMappings, $providerKey);
|
||||||
|
|
||||||
if (empty($keywords)) {
|
if (empty($keywords)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$providerResults = $provider->searchByKeywordsBatch($keywords);
|
$providerResults = $provider->searchByKeywordsBatch($keywords);
|
||||||
|
|
||||||
// Map results back to parts
|
// Map results back to parts
|
||||||
foreach ($parts as $part) {
|
foreach ($parts as $part) {
|
||||||
foreach ($fieldMappings as $mapping) {
|
foreach ($fieldMappings as $mapping) {
|
||||||
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
|
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
$keyword = $this->getKeywordFromField($part, $mapping['field']);
|
$keyword = $this->getKeywordFromField($part, $mapping['field']);
|
||||||
if ($keyword && isset($providerResults[$keyword])) {
|
if ($keyword && isset($providerResults[$keyword])) {
|
||||||
foreach ($providerResults[$keyword] as $dto) {
|
foreach ($providerResults[$keyword] as $dto) {
|
||||||
|
|
@ -145,13 +141,20 @@ final class BulkInfoProviderService
|
||||||
return $batchResults;
|
return $batchResults;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Part[] $parts
|
||||||
|
* @param array $fieldMappings
|
||||||
|
* @param array $regularProviders
|
||||||
|
* @param array $excludeResults
|
||||||
|
* @return array
|
||||||
|
*/
|
||||||
private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array
|
private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array
|
||||||
{
|
{
|
||||||
$regularResults = [];
|
$regularResults = [];
|
||||||
|
|
||||||
foreach ($parts as $part) {
|
foreach ($parts as $part) {
|
||||||
$regularResults[$part->getId()] = [];
|
$regularResults[$part->getId()] = [];
|
||||||
|
|
||||||
// Skip if we already have batch results for this part
|
// Skip if we already have batch results for this part
|
||||||
if (!empty($excludeResults[$part->getId()] ?? [])) {
|
if (!empty($excludeResults[$part->getId()] ?? [])) {
|
||||||
continue;
|
continue;
|
||||||
|
|
@ -160,7 +163,7 @@ final class BulkInfoProviderService
|
||||||
foreach ($fieldMappings as $mapping) {
|
foreach ($fieldMappings as $mapping) {
|
||||||
$field = $mapping['field'];
|
$field = $mapping['field'];
|
||||||
$providers = array_intersect($mapping['providers'] ?? [], array_keys($regularProviders));
|
$providers = array_intersect($mapping['providers'] ?? [], array_keys($regularProviders));
|
||||||
|
|
||||||
if (empty($providers)) {
|
if (empty($providers)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
@ -172,7 +175,7 @@ final class BulkInfoProviderService
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$dtos = $this->infoRetriever->searchByKeyword($keyword, $providers);
|
$dtos = $this->infoRetriever->searchByKeyword($keyword, $providers);
|
||||||
|
|
||||||
foreach ($dtos as $dto) {
|
foreach ($dtos as $dto) {
|
||||||
$regularResults[$part->getId()][] = new BulkSearchResultDTO(
|
$regularResults[$part->getId()][] = new BulkSearchResultDTO(
|
||||||
baseDto: $dto,
|
baseDto: $dto,
|
||||||
|
|
@ -198,13 +201,13 @@ final class BulkInfoProviderService
|
||||||
private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array
|
private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array
|
||||||
{
|
{
|
||||||
$keywords = [];
|
$keywords = [];
|
||||||
|
|
||||||
foreach ($parts as $part) {
|
foreach ($parts as $part) {
|
||||||
foreach ($fieldMappings as $mapping) {
|
foreach ($fieldMappings as $mapping) {
|
||||||
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
|
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
$keyword = $this->getKeywordFromField($part, $mapping['field']);
|
$keyword = $this->getKeywordFromField($part, $mapping['field']);
|
||||||
if ($keyword && !in_array($keyword, $keywords, true)) {
|
if ($keyword && !in_array($keyword, $keywords, true)) {
|
||||||
$keywords[] = $keyword;
|
$keywords[] = $keyword;
|
||||||
|
|
@ -251,10 +254,10 @@ final class BulkInfoProviderService
|
||||||
{
|
{
|
||||||
// Sort by priority and remove duplicates
|
// Sort by priority and remove duplicates
|
||||||
usort($bulkResults, fn($a, $b) => $a->priority <=> $b->priority);
|
usort($bulkResults, fn($a, $b) => $a->priority <=> $b->priority);
|
||||||
|
|
||||||
$uniqueResults = [];
|
$uniqueResults = [];
|
||||||
$seenKeys = [];
|
$seenKeys = [];
|
||||||
|
|
||||||
foreach ($bulkResults as $result) {
|
foreach ($bulkResults as $result) {
|
||||||
$key = "{$result->provider_key}|{$result->provider_id}";
|
$key = "{$result->provider_key}|{$result->provider_id}";
|
||||||
if (!in_array($key, $seenKeys, true)) {
|
if (!in_array($key, $seenKeys, true)) {
|
||||||
|
|
@ -294,4 +297,4 @@ final class BulkInfoProviderService
|
||||||
|
|
||||||
$this->logger->info("Prefetched details for {$prefetchCount} search results");
|
$this->logger->info("Prefetched details for {$prefetchCount} search results");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,11 +4,18 @@ declare(strict_types=1);
|
||||||
|
|
||||||
namespace App\Services\InfoProviderSystem\DTOs;
|
namespace App\Services\InfoProviderSystem\DTOs;
|
||||||
|
|
||||||
class BulkSearchRequestDTO
|
use App\Entity\Parts\Part;
|
||||||
|
|
||||||
|
readonly class BulkSearchRequestDTO
|
||||||
{
|
{
|
||||||
|
/**
|
||||||
|
* @param array $fieldMappings
|
||||||
|
* @param bool $prefetchDetails
|
||||||
|
* @param Part[] $parts The parts for which the bulk search should be performed.
|
||||||
|
*/
|
||||||
public function __construct(
|
public function __construct(
|
||||||
public readonly array $fieldMappings,
|
public array $fieldMappings,
|
||||||
public readonly bool $prefetchDetails = false,
|
public bool $prefetchDetails = false,
|
||||||
public readonly array $partIds = []
|
public array $parts = []
|
||||||
) {}
|
) {}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -664,7 +664,7 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
|
||||||
['field' => 'mpn', 'providers' => ['test'], 'priority' => 2]
|
['field' => 'mpn', 'providers' => ['test'], 'priority' => 2]
|
||||||
],
|
],
|
||||||
prefetchDetails: false,
|
prefetchDetails: false,
|
||||||
partIds: [$part->getId()]
|
parts: [$part->getId()]
|
||||||
);
|
);
|
||||||
|
|
||||||
// The service may return an empty result or throw when no results are found
|
// The service may return an empty result or throw when no results are found
|
||||||
|
|
@ -766,7 +766,7 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
|
||||||
|
|
||||||
$this->assertResponseStatusCodeSame(Response::HTTP_OK);
|
$this->assertResponseStatusCodeSame(Response::HTTP_OK);
|
||||||
|
|
||||||
// Find job from database to avoid detached entity errors
|
// Find job from database to avoid detached entity errors
|
||||||
$jobId = $job->getId();
|
$jobId = $job->getId();
|
||||||
$entityManager->clear();
|
$entityManager->clear();
|
||||||
$persistedJob = $entityManager->find(BulkInfoProviderImportJob::class, $jobId);
|
$persistedJob = $entityManager->find(BulkInfoProviderImportJob::class, $jobId);
|
||||||
|
|
@ -799,7 +799,7 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
|
||||||
['field' => 'test_supplier_spn', 'providers' => ['test'], 'priority' => 2]
|
['field' => 'test_supplier_spn', 'providers' => ['test'], 'priority' => 2]
|
||||||
],
|
],
|
||||||
prefetchDetails: false,
|
prefetchDetails: false,
|
||||||
partIds: [$part->getId()]
|
parts: [$part->getId()]
|
||||||
);
|
);
|
||||||
|
|
||||||
// The service should be able to process the request and throw an exception when no results are found
|
// The service should be able to process the request and throw an exception when no results are found
|
||||||
|
|
@ -833,7 +833,7 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
|
||||||
['field' => 'name', 'providers' => ['lcsc'], 'priority' => 1]
|
['field' => 'name', 'providers' => ['lcsc'], 'priority' => 1]
|
||||||
],
|
],
|
||||||
prefetchDetails: false,
|
prefetchDetails: false,
|
||||||
partIds: [$part->getId()]
|
parts: [$part->getId()]
|
||||||
);
|
);
|
||||||
|
|
||||||
// The service should be able to process the request and throw an exception when no results are found
|
// The service should be able to process the request and throw an exception when no results are found
|
||||||
|
|
@ -962,4 +962,4 @@ class BulkInfoProviderImportControllerTest extends WebTestCase
|
||||||
$entityManager->remove($job);
|
$entityManager->remove($job);
|
||||||
$entityManager->flush();
|
$entityManager->flush();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue