mirror of
https://github.com/Part-DB/Part-DB-server.git
synced 2026-01-14 22:29:33 +00:00
Refactor bulk info provider: replace complex arrays with DTOs
- Add BulkSearchResponseDTO, FieldMappingDTO for type safety - Use composition instead of inheritance in BulkSearchResultDTO - Remove unnecessary BulkSearchRequestDTO - Fix N+1 queries and API error handling - Fix Add Mapping button functionality
This commit is contained in:
barisgit
parent
8998b006e0
commit
2c195d9767
15 changed files with 838 additions and 195 deletions
|
|
@ -4,12 +4,14 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\Services\InfoProviderSystem;
|
||||
|
||||
use A\B;
|
||||
use App\Entity\BulkInfoProviderImportJob;
|
||||
use App\Entity\Base\AbstractPartsContainingDBElement;
|
||||
use App\Entity\Parts\Part;
|
||||
use App\Entity\Parts\Supplier;
|
||||
use App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO;
|
||||
use App\Services\InfoProviderSystem\DTOs\BulkSearchResultDTO;
|
||||
use App\Services\InfoProviderSystem\DTOs\BulkSearchResponseDTO;
|
||||
use App\Services\InfoProviderSystem\DTOs\FieldMappingDTO;
|
||||
use App\Services\InfoProviderSystem\DTOs\PartSearchResultDTO;
|
||||
use App\Services\InfoProviderSystem\DTOs\SearchResultWithMetadataDTO;
|
||||
use App\Services\InfoProviderSystem\Providers\BatchInfoProviderInterface;
|
||||
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
|
||||
use Doctrine\ORM\EntityManagerInterface;
|
||||
|
|
@ -18,6 +20,9 @@ use Symfony\Component\HttpClient\Exception\ClientException;
|
|||
|
||||
final class BulkInfoProviderService
|
||||
{
|
||||
/** @var array<string, Supplier|null> Cache for normalized supplier names */
|
||||
private array $supplierCache = [];
|
||||
|
||||
public function __construct(
|
||||
private readonly PartInfoRetriever $infoRetriever,
|
||||
private readonly ExistingPartFinder $existingPartFinder,
|
||||
|
|
@ -26,24 +31,31 @@ final class BulkInfoProviderService
|
|||
private readonly LoggerInterface $logger
|
||||
) {}
|
||||
|
||||
public function performBulkSearch(BulkSearchRequestDTO $request): array
|
||||
/**
|
||||
* Perform bulk search across multiple parts and providers.
|
||||
*
|
||||
* @param Part[] $parts Array of parts to search for
|
||||
* @param FieldMappingDTO[] $fieldMappings Array of field mappings defining search strategy
|
||||
* @param bool $prefetchDetails Whether to prefetch detailed information for results
|
||||
* @return BulkSearchResponseDTO Structured response containing all search results
|
||||
* @throws \InvalidArgumentException If no valid parts provided
|
||||
* @throws \RuntimeException If no search results found for any parts
|
||||
*/
|
||||
public function performBulkSearch(array $parts, array $fieldMappings, bool $prefetchDetails = false): BulkSearchResponseDTO
|
||||
{
|
||||
$parts = $request->parts;
|
||||
|
||||
if (empty($parts)) {
|
||||
throw new \InvalidArgumentException('No valid parts found for bulk import');
|
||||
}
|
||||
|
||||
$searchResults = [];
|
||||
$partResults = [];
|
||||
$hasAnyResults = false;
|
||||
|
||||
// Group providers by batch capability
|
||||
$batchProviders = [];
|
||||
$regularProviders = [];
|
||||
|
||||
foreach ($request->fieldMappings as $mapping) {
|
||||
$providers = $mapping['providers'] ?? [];
|
||||
foreach ($providers as $providerKey) {
|
||||
foreach ($fieldMappings as $mapping) {
|
||||
foreach ($mapping->providers as $providerKey) {
|
||||
if (!is_string($providerKey)) {
|
||||
$this->logger->error('Invalid provider key type', [
|
||||
'providerKey' => $providerKey,
|
||||
|
|
@ -62,18 +74,14 @@ final class BulkInfoProviderService
|
|||
}
|
||||
|
||||
// Process batch providers first (more efficient)
|
||||
$batchResults = $this->processBatchProviders($parts, $request->fieldMappings, $batchProviders);
|
||||
$batchResults = $this->processBatchProviders($parts, $fieldMappings, $batchProviders);
|
||||
|
||||
// Process regular providers
|
||||
$regularResults = $this->processRegularProviders($parts, $request->fieldMappings, $regularProviders, $batchResults);
|
||||
$regularResults = $this->processRegularProviders($parts, $fieldMappings, $regularProviders, $batchResults);
|
||||
|
||||
// Combine and format results
|
||||
// Combine and format results for each part
|
||||
foreach ($parts as $part) {
|
||||
$partResult = [
|
||||
'part' => $part,
|
||||
'search_results' => [],
|
||||
'errors' => []
|
||||
];
|
||||
$searchResults = [];
|
||||
|
||||
// Get results from batch and regular processing
|
||||
$allResults = array_merge(
|
||||
|
|
@ -83,24 +91,37 @@ final class BulkInfoProviderService
|
|||
|
||||
if (!empty($allResults)) {
|
||||
$hasAnyResults = true;
|
||||
$partResult['search_results'] = $this->formatSearchResults($allResults);
|
||||
$searchResults = $this->formatSearchResults($allResults);
|
||||
}
|
||||
|
||||
$searchResults[] = $partResult;
|
||||
$partResults[] = new PartSearchResultDTO(
|
||||
part: $part,
|
||||
searchResults: $searchResults,
|
||||
errors: []
|
||||
);
|
||||
}
|
||||
|
||||
if (!$hasAnyResults) {
|
||||
throw new \RuntimeException('No search results found for any of the selected parts');
|
||||
}
|
||||
|
||||
return $searchResults;
|
||||
$response = new BulkSearchResponseDTO($partResults);
|
||||
|
||||
// Prefetch details if requested
|
||||
if ($prefetchDetails) {
|
||||
$this->prefetchDetailsForResults($response);
|
||||
}
|
||||
|
||||
return $response;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Part[] $parts
|
||||
* @param array $fieldMappings
|
||||
* @param array<string, BatchInfoProviderInterface> $batchProviders
|
||||
* @return array<int, BulkSearchResultDTO[]> A list of results indexed by part ID
|
||||
* Process parts using batch-capable info providers.
|
||||
*
|
||||
* @param Part[] $parts Array of parts to search for
|
||||
* @param FieldMappingDTO[] $fieldMappings Array of field mapping configurations
|
||||
* @param array<string, BatchInfoProviderInterface> $batchProviders Batch providers indexed by key
|
||||
* @return array<int, BulkSearchResultDTO[]> Results indexed by part ID
|
||||
*/
|
||||
private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array
|
||||
{
|
||||
|
|
@ -119,19 +140,19 @@ final class BulkInfoProviderService
|
|||
// Map results back to parts
|
||||
foreach ($parts as $part) {
|
||||
foreach ($fieldMappings as $mapping) {
|
||||
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
|
||||
if (!in_array($providerKey, $mapping->providers, true)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$keyword = $this->getKeywordFromField($part, $mapping['field']);
|
||||
$keyword = $this->getKeywordFromField($part, $mapping->field);
|
||||
if ($keyword && isset($providerResults[$keyword])) {
|
||||
foreach ($providerResults[$keyword] as $dto) {
|
||||
$batchResults[$part->getId()][] = new BulkSearchResultDTO(
|
||||
baseDto: $dto,
|
||||
sourceField: $mapping['field'],
|
||||
sourceField: $mapping->field,
|
||||
sourceKeyword: $keyword,
|
||||
localPart: $this->existingPartFinder->findFirstExisting($dto),
|
||||
priority: $mapping['priority'] ?? 1
|
||||
priority: $mapping->priority
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -149,11 +170,13 @@ final class BulkInfoProviderService
|
|||
}
|
||||
|
||||
/**
|
||||
* @param Part[] $parts
|
||||
* @param array $fieldMappings
|
||||
* @param array<string, InfoProviderInterface> $regularProviders The info providers that do not support batch searching, indexed by their provider key
|
||||
* @param array $excludeResults
|
||||
* @return array <int, BulkSearchResultDTO[]> A list of results indexed by part ID
|
||||
* Process parts using regular (non-batch) info providers.
|
||||
*
|
||||
* @param Part[] $parts Array of parts to search for
|
||||
* @param FieldMappingDTO[] $fieldMappings Array of field mapping configurations
|
||||
* @param array<string, InfoProviderInterface> $regularProviders Regular providers indexed by key
|
||||
* @param array<int, BulkSearchResultDTO[]> $excludeResults Results to exclude (from batch processing)
|
||||
* @return array<int, BulkSearchResultDTO[]> Results indexed by part ID
|
||||
*/
|
||||
private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array
|
||||
{
|
||||
|
|
@ -168,14 +191,13 @@ final class BulkInfoProviderService
|
|||
}
|
||||
|
||||
foreach ($fieldMappings as $mapping) {
|
||||
$field = $mapping['field'];
|
||||
$providers = array_intersect($mapping['providers'] ?? [], array_keys($regularProviders));
|
||||
$providers = array_intersect($mapping->providers, array_keys($regularProviders));
|
||||
|
||||
if (empty($providers)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$keyword = $this->getKeywordFromField($part, $field);
|
||||
$keyword = $this->getKeywordFromField($part, $mapping->field);
|
||||
if (!$keyword) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -186,16 +208,16 @@ final class BulkInfoProviderService
|
|||
foreach ($dtos as $dto) {
|
||||
$regularResults[$part->getId()][] = new BulkSearchResultDTO(
|
||||
baseDto: $dto,
|
||||
sourceField: $field,
|
||||
sourceField: $mapping->field,
|
||||
sourceKeyword: $keyword,
|
||||
localPart: $this->existingPartFinder->findFirstExisting($dto),
|
||||
priority: $mapping['priority'] ?? 1
|
||||
priority: $mapping->priority
|
||||
);
|
||||
}
|
||||
} catch (ClientException $e) {
|
||||
$this->logger->error('Regular search failed', [
|
||||
'part_id' => $part->getId(),
|
||||
'field' => $field,
|
||||
'field' => $mapping->field,
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
}
|
||||
|
|
@ -206,10 +228,12 @@ final class BulkInfoProviderService
|
|||
}
|
||||
|
||||
/**
|
||||
* @param Part[] $parts
|
||||
* @param array $fieldMappings
|
||||
* @param string $providerKey
|
||||
* @return string[]
|
||||
* Collect unique keywords for a specific provider from all parts and field mappings.
|
||||
*
|
||||
* @param Part[] $parts Array of parts to collect keywords from
|
||||
* @param FieldMappingDTO[] $fieldMappings Array of field mapping configurations
|
||||
* @param string $providerKey The provider key to collect keywords for
|
||||
* @return string[] Array of unique keywords
|
||||
*/
|
||||
private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array
|
||||
{
|
||||
|
|
@ -217,11 +241,11 @@ final class BulkInfoProviderService
|
|||
|
||||
foreach ($parts as $part) {
|
||||
foreach ($fieldMappings as $mapping) {
|
||||
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
|
||||
if (!in_array($providerKey, $mapping->providers, true)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$keyword = $this->getKeywordFromField($part, $mapping['field']);
|
||||
$keyword = $this->getKeywordFromField($part, $mapping->field);
|
||||
if ($keyword && !in_array($keyword, $keywords, true)) {
|
||||
$keywords[] = $keyword;
|
||||
}
|
||||
|
|
@ -247,22 +271,66 @@ final class BulkInfoProviderService
|
|||
}
|
||||
|
||||
$supplierKey = substr($field, 0, -4);
|
||||
$supplier = $this->getSupplierByNormalizedName($supplierKey);
|
||||
|
||||
if (!$supplier) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$orderDetail = $part->getOrderdetails()->filter(
|
||||
fn($od) => $od->getSupplier()?->getId() === $supplier->getId()
|
||||
)->first();
|
||||
|
||||
return $orderDetail !== false ? $orderDetail->getSupplierpartnr() : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get supplier by normalized name with caching to prevent N+1 queries.
|
||||
*
|
||||
* @param string $normalizedKey The normalized supplier key to search for
|
||||
* @return Supplier|null The matching supplier or null if not found
|
||||
*/
|
||||
private function getSupplierByNormalizedName(string $normalizedKey): ?Supplier
|
||||
{
|
||||
// Check cache first
|
||||
if (isset($this->supplierCache[$normalizedKey])) {
|
||||
return $this->supplierCache[$normalizedKey];
|
||||
}
|
||||
|
||||
// Use efficient database query with PHP normalization
|
||||
// Since DQL doesn't support REPLACE, we'll load all suppliers once and cache the normalization
|
||||
if (empty($this->supplierCache)) {
|
||||
$this->loadSuppliersIntoCache();
|
||||
}
|
||||
|
||||
$supplier = $this->supplierCache[$normalizedKey] ?? null;
|
||||
|
||||
// Cache the result (including null results to prevent repeated queries)
|
||||
$this->supplierCache[$normalizedKey] = $supplier;
|
||||
|
||||
return $supplier;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all suppliers into cache with normalized names to avoid N+1 queries.
|
||||
*/
|
||||
private function loadSuppliersIntoCache(): void
|
||||
{
|
||||
/** @var Supplier[] $suppliers */
|
||||
$suppliers = $this->entityManager->getRepository(Supplier::class)->findAll();
|
||||
|
||||
foreach ($suppliers as $supplier) {
|
||||
$normalizedName = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName()));
|
||||
if ($normalizedName === $supplierKey) {
|
||||
$orderDetail = $part->getOrderdetails()->filter(
|
||||
fn($od) => $od->getSupplier()?->getId() === $supplier->getId()
|
||||
)->first();
|
||||
|
||||
return $orderDetail ? $orderDetail->getSupplierpartnr() : null;
|
||||
}
|
||||
$this->supplierCache[$normalizedName] = $supplier;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format and deduplicate search results.
|
||||
*
|
||||
* @param BulkSearchResultDTO[] $bulkResults Array of bulk search results
|
||||
* @return SearchResultWithMetadataDTO[] Array of formatted search results with metadata
|
||||
*/
|
||||
private function formatSearchResults(array $bulkResults): array
|
||||
{
|
||||
// Sort by priority and remove duplicates
|
||||
|
|
@ -272,38 +340,64 @@ final class BulkInfoProviderService
|
|||
$seenKeys = [];
|
||||
|
||||
foreach ($bulkResults as $result) {
|
||||
$key = "{$result->provider_key}|{$result->provider_id}";
|
||||
$key = "{$result->getProviderKey()}|{$result->getProviderId()}";
|
||||
if (!in_array($key, $seenKeys, true)) {
|
||||
$seenKeys[] = $key;
|
||||
$uniqueResults[] = [
|
||||
'dto' => $result,
|
||||
'localPart' => $result->localPart,
|
||||
'source_field' => $result->sourceField,
|
||||
'source_keyword' => $result->sourceKeyword
|
||||
];
|
||||
$uniqueResults[] = new SearchResultWithMetadataDTO(
|
||||
searchResult: $result,
|
||||
localPart: $result->localPart,
|
||||
sourceField: $result->sourceField,
|
||||
sourceKeyword: $result->sourceKeyword
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return $uniqueResults;
|
||||
}
|
||||
|
||||
public function prefetchDetailsForResults(array $searchResults): void
|
||||
/**
|
||||
* Prefetch detailed information for search results.
|
||||
*
|
||||
* @param BulkSearchResponseDTO|array $searchResults Search results (supports both new DTO and legacy array format)
|
||||
*/
|
||||
public function prefetchDetailsForResults($searchResults): void
|
||||
{
|
||||
$prefetchCount = 0;
|
||||
|
||||
foreach ($searchResults as $partResult) {
|
||||
foreach ($partResult['search_results'] as $result) {
|
||||
$dto = $result['dto'];
|
||||
// Handle both new DTO format and legacy array format for backwards compatibility
|
||||
if ($searchResults instanceof BulkSearchResponseDTO) {
|
||||
foreach ($searchResults->partResults as $partResult) {
|
||||
foreach ($partResult->searchResults as $result) {
|
||||
$dto = $result->searchResult;
|
||||
|
||||
try {
|
||||
$this->infoRetriever->getDetails($dto->provider_key, $dto->provider_id);
|
||||
$prefetchCount++;
|
||||
} catch (\Exception $e) {
|
||||
$this->logger->warning('Failed to prefetch details for provider part', [
|
||||
'provider_key' => $dto->provider_key,
|
||||
'provider_id' => $dto->provider_id,
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
try {
|
||||
$this->infoRetriever->getDetails($dto->getProviderKey(), $dto->getProviderId());
|
||||
$prefetchCount++;
|
||||
} catch (\Exception $e) {
|
||||
$this->logger->warning('Failed to prefetch details for provider part', [
|
||||
'provider_key' => $dto->getProviderKey(),
|
||||
'provider_id' => $dto->getProviderId(),
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Legacy array format support
|
||||
foreach ($searchResults as $partResult) {
|
||||
foreach ($partResult['search_results'] as $result) {
|
||||
$dto = $result['dto'];
|
||||
|
||||
try {
|
||||
$this->infoRetriever->getDetails($dto->getProviderKey(), $dto->getProviderId());
|
||||
$prefetchCount++;
|
||||
} catch (\Exception $e) {
|
||||
$this->logger->warning('Failed to prefetch details for provider part', [
|
||||
'provider_key' => $dto->getProviderKey(),
|
||||
'provider_id' => $dto->getProviderId(),
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue