groove/Part-DB-server
1
0
Fork 0
mirror of https://github.com/Part-DB/Part-DB-server.git synced 2025-12-10 04:59:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Refactor bulk import functionality to make controller smaller (use services) add DTOs and use stimulus controllers on frontend

Browse source
This commit is contained in:
barisgit 2025-09-09 20:30:27 +02:00
parent 65d840c444
commit d6ac16ede0
14 changed files with 1382 additions and 716 deletions
Download patch file Download diff file Expand all files Collapse all files

591
src/Controller/BulkInfoProviderImportController.php
View file

@ -28,16 +28,14 @@ use App\Entity\BulkImportJobStatus;
use App\Entity\Parts\Part;
use App\Entity\Parts\Supplier;
use App\Form\InfoProviderSystem\GlobalFieldMappingType;
use App\Services\InfoProviderSystem\PartInfoRetriever;
use App\Services\InfoProviderSystem\ExistingPartFinder;
use App\Services\InfoProviderSystem\ProviderRegistry;
use App\Services\InfoProviderSystem\Providers\LCSCProvider;
use App\Services\InfoProviderSystem\BulkInfoProviderService;
use App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\HttpClient\Exception\ClientException;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\Routing\Attribute\Route;
use App\Entity\UserSystem\User;
@ -45,20 +43,67 @@ use App\Entity\UserSystem\User;
class BulkInfoProviderImportController extends AbstractController
{
public function __construct(
private readonly PartInfoRetriever $infoRetriever,
private readonly LCSCProvider $LCSCProvider,
private readonly ExistingPartFinder $existingPartFinder,
private readonly EntityManagerInterface $entityManager
private readonly BulkInfoProviderService $bulkService,
private readonly EntityManagerInterface $entityManager,
private readonly LoggerInterface $logger
) {
}
private function createErrorResponse(string $message, int $statusCode = 400, array $context = []): JsonResponse
{
$this->logger->warning('Bulk import operation failed', array_merge([
'error' => $message,
'user' => $this->getUser()?->getUserIdentifier(),
], $context));
return $this->json([
'success' => false,
'error' => $message
], $statusCode);
}
private function validateJobAccess(int $jobId): ?BulkInfoProviderImportJob
{
$job = $this->entityManager->getRepository(BulkInfoProviderImportJob::class)->find($jobId);
if (!$job) {
return null;
}
if ($job->getCreatedBy() !== $this->getUser()) {
return null;
}
return $job;
}
private function updatePartSearchResults(BulkInfoProviderImportJob $job, int $partId, ?array $newResults): void
{
if ($newResults === null) {
return;
}
// Only deserialize and update if we have new results
$allResults = $job->deserializeSearchResults($this->entityManager);
// Find and update the results for this specific part
foreach ($allResults as $index => $partResult) {
if ($partResult['part']->getId() === $partId) {
$allResults[$index] = $newResults;
break;
}
}
// Save updated results back to job
$job->setSearchResults($job->serializeSearchResults($allResults));
}
#[Route('/step1', name: 'bulk_info_provider_step1')]
public function step1(Request $request, LoggerInterface $exceptionLogger): Response
public function step1(Request $request): Response
{
$this->denyAccessUnlessGranted('@info_providers.create_parts');
// Increase execution time for bulk operations
set_time_limit(600); // 10 minutes for large batches
set_time_limit(600);
$ids = $request->query->get('ids');
if (!$ids) {
@ -66,7 +111,6 @@ class BulkInfoProviderImportController extends AbstractController
return $this->redirectToRoute('homepage');
}
// Get the selected parts
$partIds = explode(',', $ids);
$partRepository = $this->entityManager->getRepository(Part::class);
$parts = $partRepository->getElementsFromIDArray($partIds);
@ -76,7 +120,6 @@ class BulkInfoProviderImportController extends AbstractController
return $this->redirectToRoute('homepage');
}
// Warn about large batches
if (count($parts) > 50) {
$this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.');
}
@ -114,23 +157,17 @@ class BulkInfoProviderImportController extends AbstractController
$fieldMappings = $formData['field_mappings'];
$prefetchDetails = $formData['prefetch_details'] ?? false;
// Debug logging
$exceptionLogger->info('Form data received', [
'prefetch_details' => $prefetchDetails,
'prefetch_details_type' => gettype($prefetchDetails)
]);
$user = $this->getUser();
if (!$user instanceof User) {
throw new \RuntimeException('User must be authenticated and of type User');
}
// Create and save the job
$job = new BulkInfoProviderImportJob();
$job->setFieldMappings($fieldMappings);
$job->setPrefetchDetails($prefetchDetails);
$user = $this->getUser();
if (!$user instanceof User) {
throw new \RuntimeException('User must be authenticated and of type User');
}
$job->setCreatedBy($user);
// Create job parts for each part
foreach ($parts as $part) {
$jobPart = new BulkInfoProviderImportJobPart($job, $part);
$job->addJobPart($jobPart);
@ -139,200 +176,40 @@ class BulkInfoProviderImportController extends AbstractController
$this->entityManager->persist($job);
$this->entityManager->flush();
$searchResults = [];
$hasAnyResults = false;
try {
// Optimize: Use batch async requests for LCSC provider
$lcscKeywords = [];
$keywordToPartField = [];
$searchRequest = new BulkSearchRequestDTO(
fieldMappings: $fieldMappings,
prefetchDetails: $prefetchDetails,
partIds: $partIds
);
// First, collect all LCSC keywords for batch processing
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
$field = $mapping['field'];
$providers = $mapping['providers'] ?? [];
if (in_array('lcsc', $providers, true)) {
$keyword = $this->getKeywordFromField($part, $field);
if ($keyword) {
$lcscKeywords[] = $keyword;
$keywordToPartField[$keyword] = [
'part' => $part,
'field' => $field
];
}
}
}
}
// Batch search LCSC keywords asynchronously
$lcscBatchResults = [];
if (!empty($lcscKeywords)) {
try {
// Try to get LCSC provider and use batch method if available
$lcscBatchResults = $this->searchLcscBatch($lcscKeywords);
} catch (\Exception $e) {
$exceptionLogger->warning('LCSC batch search failed, falling back to individual requests', [
'error' => $e->getMessage()
]);
}
}
// Now process each part
foreach ($parts as $part) {
$partResult = [
'part' => $part,
'search_results' => [],
'errors' => []
];
// Collect all DTOs using priority-based search
$allDtos = [];
$dtoMetadata = []; // Store source field info separately
// Group mappings by priority (lower number = higher priority)
$mappingsByPriority = [];
foreach ($fieldMappings as $mapping) {
$priority = $mapping['priority'] ?? 1;
$mappingsByPriority[$priority][] = $mapping;
}
ksort($mappingsByPriority); // Sort by priority (1, 2, 3...)
// Try each priority level until we find results
foreach ($mappingsByPriority as $priority => $mappings) {
$priorityResults = [];
// For same priority, search all and combine results
foreach ($mappings as $mapping) {
$field = $mapping['field'];
$providers = $mapping['providers'] ?? [];
if (empty($providers)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $field);
if ($keyword) {
try {
// Use batch results for LCSC if available
if (in_array('lcsc', $providers, true) && isset($lcscBatchResults[$keyword])) {
$dtos = $lcscBatchResults[$keyword];
} else {
// Fall back to regular search for non-LCSC providers
$dtos = $this->infoRetriever->searchByKeyword(
keyword: $keyword,
providers: $providers
);
}
// Store field info for each DTO separately
foreach ($dtos as $dto) {
$dtoKey = $dto->provider_key . '|' . $dto->provider_id;
$dtoMetadata[$dtoKey] = [
'source_field' => $field,
'source_keyword' => $keyword,
'priority' => $priority
];
}
$priorityResults = array_merge($priorityResults, $dtos);
} catch (ClientException $e) {
$partResult['errors'][] = "Error searching with {$field} (priority {$priority}): " . $e->getMessage();
$exceptionLogger->error('Error during bulk info provider search for part ' . $part->getId() . " field {$field}: " . $e->getMessage(), ['exception' => $e]);
}
}
}
// If we found results at this priority level, use them and stop
if (!empty($priorityResults)) {
$allDtos = $priorityResults;
break;
}
}
// Remove duplicates based on provider_key + provider_id
$uniqueDtos = [];
$seenKeys = [];
foreach ($allDtos as $dto) {
if ($dto === null || !isset($dto->provider_key, $dto->provider_id)) {
continue;
}
$key = "{$dto->provider_key}|{$dto->provider_id}";
if (!in_array($key, $seenKeys, true)) {
$seenKeys[] = $key;
$uniqueDtos[] = $dto;
}
}
// Convert DTOs to result format with metadata
$partResult['search_results'] = array_map(
function ($dto) use ($dtoMetadata) {
$dtoKey = $dto->provider_key . '|' . $dto->provider_id;
$metadata = $dtoMetadata[$dtoKey] ?? [];
return [
'dto' => $dto,
'localPart' => $this->existingPartFinder->findFirstExisting($dto),
'source_field' => $metadata['source_field'] ?? null,
'source_keyword' => $metadata['source_keyword'] ?? null
];
},
$uniqueDtos
);
if (!empty($partResult['search_results'])) {
$hasAnyResults = true;
}
$searchResults[] = $partResult;
}
// Check if search was successful
if (!$hasAnyResults) {
$exceptionLogger->warning('Bulk import search returned no results for any parts', [
'job_id' => $job->getId(),
'parts_count' => count($parts)
]);
// Delete the job since it has no useful results
$this->entityManager->remove($job);
$this->entityManager->flush();
$this->addFlash('error', 'No search results found for any of the selected parts. Please check your field mappings and provider selections.');
return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
}
$searchResults = $this->bulkService->performBulkSearch($searchRequest);
// Save search results to job
$job->setSearchResults($this->serializeSearchResults($searchResults));
$job->setSearchResults($job->serializeSearchResults($searchResults));
$job->markAsInProgress();
$this->entityManager->flush();
// Prefetch details if requested
if ($prefetchDetails) {
$this->bulkService->prefetchDetailsForResults($searchResults);
}
return $this->redirectToRoute('bulk_info_provider_step2', ['jobId' => $job->getId()]);
} catch (\Exception $e) {
$exceptionLogger->error('Critical error during bulk import search', [
$this->logger->error('Critical error during bulk import search', [
'job_id' => $job->getId(),
'error' => $e->getMessage(),
'exception' => $e
]);
// Delete the job on critical failure
$this->entityManager->remove($job);
$this->entityManager->flush();
$this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage());
return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
}
// Prefetch details if requested
if ($prefetchDetails) {
$exceptionLogger->info('Prefetch details requested, starting prefetch for ' . count($searchResults) . ' parts');
$this->prefetchDetailsForResults($searchResults, $exceptionLogger);
} else {
$exceptionLogger->info('Prefetch details not requested, skipping prefetch');
}
// Redirect to step 2 with the job
return $this->redirectToRoute('bulk_info_provider_step2', ['jobId' => $job->getId()]);
}
// Get existing in-progress jobs for current user
@ -433,72 +310,6 @@ class BulkInfoProviderImportController extends AbstractController
return $this->json(['success' => true]);
}
private function getKeywordFromField(Part $part, string $field): ?string
{
return match ($field) {
'mpn' => $part->getManufacturerProductNumber(),
'name' => $part->getName(),
default => $this->getSupplierPartNumber($part, $field)
};
}
private function getSupplierPartNumber(Part $part, string $field): ?string
{
// Check if this is a supplier SPN field
if (!str_ends_with($field, '_spn')) {
return null;
}
// Extract supplier key (remove _spn suffix)
$supplierKey = substr($field, 0, -4);
// Get all suppliers to find matching one
$suppliers = $this->entityManager->getRepository(Supplier::class)->findAll();
foreach ($suppliers as $supplier) {
$normalizedName = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName()));
if ($normalizedName === $supplierKey) {
// Find order detail for this supplier
$orderDetail = $part->getOrderdetails()->filter(
fn($od) => $od->getSupplier()?->getId() === $supplier->getId()
)->first();
return $orderDetail ? $orderDetail->getSupplierpartnr() : null;
}
}
return null;
}
/**
* Prefetch details for all search results to populate cache
*/
private function prefetchDetailsForResults(array $searchResults, LoggerInterface $logger): void
{
$prefetchCount = 0;
foreach ($searchResults as $partResult) {
foreach ($partResult['search_results'] as $result) {
$dto = $result['dto'];
try {
// This call will cache the details for later use
$this->infoRetriever->getDetails($dto->provider_key, $dto->provider_id);
$prefetchCount++;
} catch (\Exception $e) {
$logger->warning('Failed to prefetch details for provider part', [
'provider_key' => $dto->provider_key,
'provider_id' => $dto->provider_id,
'error' => $e->getMessage()
]);
}
}
}
if ($prefetchCount > 0) {
$this->addFlash('success', "Prefetched details for {$prefetchCount} search results");
}
}
#[Route('/step2/{jobId}', name: 'bulk_info_provider_step2')]
public function step2(int $jobId): Response
@ -518,7 +329,7 @@ class BulkInfoProviderImportController extends AbstractController
// Get the parts and deserialize search results
$parts = $job->getJobParts()->map(fn($jobPart) => $jobPart->getPart())->toArray();
$searchResults = $this->deserializeSearchResults($job->getSearchResults(), $parts);
$searchResults = $job->deserializeSearchResults($this->entityManager);
return $this->render('info_providers/bulk_import/step2.html.twig', [
'job' => $job,
@ -527,103 +338,6 @@ class BulkInfoProviderImportController extends AbstractController
]);
}
private function serializeSearchResults(array $searchResults): array
{
$serialized = [];
foreach ($searchResults as $partResult) {
$partData = [
'part_id' => $partResult['part']->getId(),
'search_results' => [],
'errors' => $partResult['errors']
];
foreach ($partResult['search_results'] as $result) {
$dto = $result['dto'];
$partData['search_results'][] = [
'dto' => [
'provider_key' => $dto->provider_key,
'provider_id' => $dto->provider_id,
'name' => $dto->name,
'description' => $dto->description,
'manufacturer' => $dto->manufacturer,
'mpn' => $dto->mpn,
'provider_url' => $dto->provider_url,
'preview_image_url' => $dto->preview_image_url,
'_source_field' => $result['source_field'] ?? null,
'_source_keyword' => $result['source_keyword'] ?? null,
],
'localPart' => $result['localPart'] ? $result['localPart']->getId() : null
];
}
$serialized[] = $partData;
}
return $serialized;
}
private function deserializeSearchResults(array $serializedResults, array $parts): array
{
$partsById = [];
foreach ($parts as $part) {
$partsById[$part->getId()] = $part;
}
$searchResults = [];
foreach ($serializedResults as $partData) {
$part = $partsById[$partData['part_id']] ?? null;
if (!$part) {
continue;
}
$partResult = [
'part' => $part,
'search_results' => [],
'errors' => $partData['errors']
];
foreach ($partData['search_results'] as $resultData) {
$dtoData = $resultData['dto'];
$dto = new \App\Services\InfoProviderSystem\DTOs\SearchResultDTO(
provider_key: $dtoData['provider_key'],
provider_id: $dtoData['provider_id'],
name: $dtoData['name'],
description: $dtoData['description'],
manufacturer: $dtoData['manufacturer'],
mpn: $dtoData['mpn'],
provider_url: $dtoData['provider_url'],
preview_image_url: $dtoData['preview_image_url']
);
$localPart = null;
if ($resultData['localPart']) {
$localPart = $this->entityManager->getRepository(Part::class)->find($resultData['localPart']);
}
$partResult['search_results'][] = [
'dto' => $dto,
'localPart' => $localPart,
'source_field' => $dtoData['_source_field'] ?? null,
'source_keyword' => $dtoData['_source_keyword'] ?? null
];
}
$searchResults[] = $partResult;
}
return $searchResults;
}
/**
* Perform batch LCSC search using async HTTP requests
*/
private function searchLcscBatch(array $keywords): array
{
return $this->LCSCProvider->searchByKeywordsBatch($keywords);
}
#[Route('/job/{jobId}/part/{partId}/mark-completed', name: 'bulk_info_provider_mark_completed', methods: ['POST'])]
public function markPartCompleted(int $jobId, int $partId): Response
@ -702,4 +416,155 @@ class BulkInfoProviderImportController extends AbstractController
'job_completed' => $job->isCompleted()
]);
}
#[Route('/job/{jobId}/part/{partId}/research', name: 'bulk_info_provider_research_part', methods: ['POST'])]
public function researchPart(int $jobId, int $partId): JsonResponse
{
$job = $this->validateJobAccess($jobId);
if (!$job) {
return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]);
}
$part = $this->entityManager->getRepository(Part::class)->find($partId);
if (!$part) {
return $this->createErrorResponse('Part not found', 404, ['part_id' => $partId]);
}
// Only refresh if the entity might be stale (optional optimization)
if ($this->entityManager->getUnitOfWork()->isScheduledForUpdate($part)) {
$this->entityManager->refresh($part);
}
try {
// Use the job's field mappings to perform the search
$fieldMappings = $job->getFieldMappings();
$prefetchDetails = $job->isPrefetchDetails();
$searchRequest = new BulkSearchRequestDTO(
fieldMappings: $fieldMappings,
prefetchDetails: $prefetchDetails,
partIds: [$partId]
);
try {
$searchResults = $this->bulkService->performBulkSearch($searchRequest);
} catch (\Exception $searchException) {
// Handle "no search results found" as a normal case, not an error
if (str_contains($searchException->getMessage(), 'No search results found')) {
$searchResults = [];
} else {
throw $searchException;
}
}
// Update the job's search results for this specific part efficiently
$this->updatePartSearchResults($job, $partId, $searchResults[0] ?? null);
// Prefetch details if requested
if ($prefetchDetails && !empty($searchResults)) {
$this->bulkService->prefetchDetailsForResults($searchResults);
}
$this->entityManager->flush();
// Return the new results for this part
$newResults = $searchResults[0] ?? null;
return $this->json([
'success' => true,
'part_id' => $partId,
'results_count' => $newResults ? count($newResults['search_results']) : 0,
'errors_count' => $newResults ? count($newResults['errors']) : 0,
'message' => 'Part research completed successfully'
]);
} catch (\Exception $e) {
return $this->createErrorResponse(
'Research failed: ' . $e->getMessage(),
500,
[
'job_id' => $jobId,
'part_id' => $partId,
'exception' => $e->getMessage()
]
);
}
}
#[Route('/job/{jobId}/research-all', name: 'bulk_info_provider_research_all', methods: ['POST'])]
public function researchAllParts(int $jobId): JsonResponse
{
$job = $this->validateJobAccess($jobId);
if (!$job) {
return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]);
}
// Get all part IDs that are not completed or skipped
$partIds = [];
foreach ($job->getJobParts() as $jobPart) {
if (!$jobPart->isCompleted() && !$jobPart->isSkipped()) {
$partIds[] = $jobPart->getPart()->getId();
}
}
if (empty($partIds)) {
return $this->json([
'success' => true,
'message' => 'No parts to research',
'researched_count' => 0
]);
}
try {
$fieldMappings = $job->getFieldMappings();
$prefetchDetails = $job->isPrefetchDetails();
// Process in batches to reduce memory usage for large operations
$batchSize = 20; // Configurable batch size for memory management
$allResults = [];
$batches = array_chunk($partIds, $batchSize);
foreach ($batches as $batch) {
$searchRequest = new BulkSearchRequestDTO(
fieldMappings: $fieldMappings,
prefetchDetails: $prefetchDetails,
partIds: $batch
);
$batchResults = $this->bulkService->performBulkSearch($searchRequest);
$allResults = array_merge($allResults, $batchResults);
// Clear entity manager periodically to prevent memory issues
$this->entityManager->clear();
$job = $this->entityManager->find(BulkInfoProviderImportJob::class, $job->getId());
}
// Update the job's search results
$job->setSearchResults($job->serializeSearchResults($allResults));
// Prefetch details if requested
if ($prefetchDetails) {
$this->bulkService->prefetchDetailsForResults($allResults);
}
$this->entityManager->flush();
return $this->json([
'success' => true,
'researched_count' => count($partIds),
'message' => sprintf('Successfully researched %d parts', count($partIds))
]);
} catch (\Exception $e) {
return $this->createErrorResponse(
'Bulk research failed: ' . $e->getMessage(),
500,
[
'job_id' => $jobId,
'part_ids' => $partIds,
'exception' => $e->getMessage()
]
);
}
}
}

97
src/Entity/BulkInfoProviderImportJob.php
View file

@ -25,10 +25,12 @@ namespace App\Entity;
use App\Entity\Base\AbstractDBElement;
use App\Entity\Parts\Part;
use App\Entity\UserSystem\User;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
use Doctrine\Common\Collections\ArrayCollection;
use Doctrine\Common\Collections\Collection;
use Doctrine\DBAL\Types\Types;
use Doctrine\ORM\Mapping as ORM;
use Doctrine\ORM\EntityManagerInterface;
enum BulkImportJobStatus: string
{
@ -403,4 +405,99 @@ class BulkInfoProviderImportJob extends AbstractDBElement
$completed = $this->getCompletedPartsCount() + $this->getSkippedPartsCount();
return $completed >= $total;
}
public function serializeSearchResults(array $searchResults): array
{
$serialized = [];
foreach ($searchResults as $partResult) {
$partData = [
'part_id' => $partResult['part']->getId(),
'search_results' => [],
'errors' => $partResult['errors'] ?? []
];
foreach ($partResult['search_results'] as $result) {
$dto = $result['dto'];
$partData['search_results'][] = [
'dto' => [
'provider_key' => $dto->provider_key,
'provider_id' => $dto->provider_id,
'name' => $dto->name,
'description' => $dto->description,
'manufacturer' => $dto->manufacturer,
'mpn' => $dto->mpn,
'provider_url' => $dto->provider_url,
'preview_image_url' => $dto->preview_image_url,
'_source_field' => $result['source_field'] ?? null,
'_source_keyword' => $result['source_keyword'] ?? null,
],
'localPart' => $result['localPart'] ? $result['localPart']->getId() : null
];
}
$serialized[] = $partData;
}
return $serialized;
}
public function deserializeSearchResults(?EntityManagerInterface $entityManager = null): array
{
if (empty($this->searchResults)) {
return [];
}
$parts = $this->jobParts->map(fn($jobPart) => $jobPart->getPart())->toArray();
$partsById = [];
foreach ($parts as $part) {
$partsById[$part->getId()] = $part;
}
$searchResults = [];
foreach ($this->searchResults as $partData) {
$part = $partsById[$partData['part_id']] ?? null;
if (!$part) {
continue;
}
$partResult = [
'part' => $part,
'search_results' => [],
'errors' => $partData['errors'] ?? []
];
foreach ($partData['search_results'] as $resultData) {
$dtoData = $resultData['dto'];
$dto = new SearchResultDTO(
provider_key: $dtoData['provider_key'],
provider_id: $dtoData['provider_id'],
name: $dtoData['name'],
description: $dtoData['description'],
manufacturer: $dtoData['manufacturer'],
mpn: $dtoData['mpn'],
provider_url: $dtoData['provider_url'],
preview_image_url: $dtoData['preview_image_url']
);
$localPart = null;
if ($resultData['localPart'] && $entityManager) {
$localPart = $entityManager->getRepository(Part::class)->find($resultData['localPart']);
}
$partResult['search_results'][] = [
'dto' => $dto,
'localPart' => $localPart,
'source_field' => $dtoData['_source_field'] ?? null,
'source_keyword' => $dtoData['_source_keyword'] ?? null
];
}
$searchResults[] = $partResult;
}
return $searchResults;
}
}

2
src/Form/InfoProviderSystem/GlobalFieldMappingType.php
View file

@ -54,7 +54,7 @@ class GlobalFieldMappingType extends AbstractType
]);
$builder->add('submit', SubmitType::class, [
'label' => 'info_providers.bulk_search.submit'
'label' => 'info_providers.bulk_import.search.submit'
]);
}

15
src/Form/InfoProviderSystem/ProviderSelectType.php
View file

@ -24,9 +24,7 @@ declare(strict_types=1);
namespace App\Form\InfoProviderSystem;
use App\Services\InfoProviderSystem\ProviderRegistry;
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
use Symfony\Component\Form\AbstractType;
use Symfony\Component\Form\ChoiceList\ChoiceList;
use Symfony\Component\Form\Extension\Core\Type\ChoiceType;
use Symfony\Component\OptionsResolver\OptionsResolver;
@ -44,11 +42,16 @@ class ProviderSelectType extends AbstractType
public function configureOptions(OptionsResolver $resolver): void
{
$providers = $this->providerRegistry->getActiveProviders();
// Create a simple array of provider keys => labels
$choices = [];
foreach ($providers as $provider) {
$choices[$provider->getProviderInfo()['name']] = $provider->getProviderKey();
}
$resolver->setDefaults([
'choices' => $this->providerRegistry->getActiveProviders(),
'choice_label' => ChoiceList::label($this, static fn (?InfoProviderInterface $choice) => $choice?->getProviderInfo()['name']),
'choice_value' => ChoiceList::value($this, static fn(?InfoProviderInterface $choice) => $choice?->getProviderKey()),
'choices' => $choices,
'multiple' => true,
]);
}

297
src/Services/InfoProviderSystem/BulkInfoProviderService.php Normal file
View file

@ -0,0 +1,297 @@
<?php
declare(strict_types=1);
namespace App\Services\InfoProviderSystem;
use App\Entity\BulkInfoProviderImportJob;
use App\Entity\Parts\Part;
use App\Entity\Parts\Supplier;
use App\Services\InfoProviderSystem\DTOs\BulkSearchRequestDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchResultDTO;
use App\Services\InfoProviderSystem\Providers\BatchInfoProviderInterface;
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\HttpClient\Exception\ClientException;
final class BulkInfoProviderService
{
public function __construct(
private readonly PartInfoRetriever $infoRetriever,
private readonly ExistingPartFinder $existingPartFinder,
private readonly ProviderRegistry $providerRegistry,
private readonly EntityManagerInterface $entityManager,
private readonly LoggerInterface $logger
) {}
public function performBulkSearch(BulkSearchRequestDTO $request): array
{
// Convert string IDs to integers
$partIds = array_map('intval', $request->partIds);
$partRepository = $this->entityManager->getRepository(Part::class);
$parts = $partRepository->getElementsFromIDArray($partIds);
if (empty($parts)) {
throw new \InvalidArgumentException('No valid parts found for bulk import');
}
$searchResults = [];
$hasAnyResults = false;
// Group providers by batch capability
$batchProviders = [];
$regularProviders = [];
foreach ($request->fieldMappings as $mapping) {
$providers = $mapping['providers'] ?? [];
foreach ($providers as $providerKey) {
if (!is_string($providerKey)) {
$this->logger->error('Invalid provider key type', [
'providerKey' => $providerKey,
'type' => gettype($providerKey)
]);
continue;
}
$provider = $this->providerRegistry->getProviderByKey($providerKey);
if ($provider instanceof BatchInfoProviderInterface) {
$batchProviders[$providerKey] = $provider;
} else {
$regularProviders[$providerKey] = $provider;
}
}
}
// Process batch providers first (more efficient)
$batchResults = $this->processBatchProviders($parts, $request->fieldMappings, $batchProviders);
// Process regular providers
$regularResults = $this->processRegularProviders($parts, $request->fieldMappings, $regularProviders, $batchResults);
// Combine and format results
foreach ($parts as $part) {
$partResult = [
'part' => $part,
'search_results' => [],
'errors' => []
];
// Get results from batch and regular processing
$allResults = array_merge(
$batchResults[$part->getId()] ?? [],
$regularResults[$part->getId()] ?? []
);
if (!empty($allResults)) {
$hasAnyResults = true;
$partResult['search_results'] = $this->formatSearchResults($allResults);
}
$searchResults[] = $partResult;
}
if (!$hasAnyResults) {
throw new \RuntimeException('No search results found for any of the selected parts');
}
return $searchResults;
}
private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array
{
$batchResults = [];
foreach ($batchProviders as $providerKey => $provider) {
$keywords = $this->collectKeywordsForProvider($parts, $fieldMappings, $providerKey);
if (empty($keywords)) {
continue;
}
try {
$providerResults = $provider->searchByKeywordsBatch($keywords);
// Map results back to parts
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping['field']);
if ($keyword && isset($providerResults[$keyword])) {
foreach ($providerResults[$keyword] as $dto) {
$batchResults[$part->getId()][] = new BulkSearchResultDTO(
baseDto: $dto,
sourceField: $mapping['field'],
sourceKeyword: $keyword,
localPart: $this->existingPartFinder->findFirstExisting($dto),
priority: $mapping['priority'] ?? 1
);
}
}
}
}
} catch (\Exception $e) {
$this->logger->error('Batch search failed for provider ' . $providerKey, [
'error' => $e->getMessage(),
'provider' => $providerKey
]);
}
}
return $batchResults;
}
private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array
{
$regularResults = [];
foreach ($parts as $part) {
$regularResults[$part->getId()] = [];
// Skip if we already have batch results for this part
if (!empty($excludeResults[$part->getId()] ?? [])) {
continue;
}
foreach ($fieldMappings as $mapping) {
$field = $mapping['field'];
$providers = array_intersect($mapping['providers'] ?? [], array_keys($regularProviders));
if (empty($providers)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $field);
if (!$keyword) {
continue;
}
try {
$dtos = $this->infoRetriever->searchByKeyword($keyword, $providers);
foreach ($dtos as $dto) {
$regularResults[$part->getId()][] = new BulkSearchResultDTO(
baseDto: $dto,
sourceField: $field,
sourceKeyword: $keyword,
localPart: $this->existingPartFinder->findFirstExisting($dto),
priority: $mapping['priority'] ?? 1
);
}
} catch (ClientException $e) {
$this->logger->error('Regular search failed', [
'part_id' => $part->getId(),
'field' => $field,
'error' => $e->getMessage()
]);
}
}
}
return $regularResults;
}
private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array
{
$keywords = [];
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping['providers'] ?? [], true)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping['field']);
if ($keyword && !in_array($keyword, $keywords, true)) {
$keywords[] = $keyword;
}
}
}
return $keywords;
}
private function getKeywordFromField(Part $part, string $field): ?string
{
return match ($field) {
'mpn' => $part->getManufacturerProductNumber(),
'name' => $part->getName(),
default => $this->getSupplierPartNumber($part, $field)
};
}
private function getSupplierPartNumber(Part $part, string $field): ?string
{
if (!str_ends_with($field, '_spn')) {
return null;
}
$supplierKey = substr($field, 0, -4);
$suppliers = $this->entityManager->getRepository(Supplier::class)->findAll();
foreach ($suppliers as $supplier) {
$normalizedName = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName()));
if ($normalizedName === $supplierKey) {
$orderDetail = $part->getOrderdetails()->filter(
fn($od) => $od->getSupplier()?->getId() === $supplier->getId()
)->first();
return $orderDetail ? $orderDetail->getSupplierpartnr() : null;
}
}
return null;
}
private function formatSearchResults(array $bulkResults): array
{
// Sort by priority and remove duplicates
usort($bulkResults, fn($a, $b) => $a->priority <=> $b->priority);
$uniqueResults = [];
$seenKeys = [];
foreach ($bulkResults as $result) {
$key = "{$result->provider_key}|{$result->provider_id}";
if (!in_array($key, $seenKeys, true)) {
$seenKeys[] = $key;
$uniqueResults[] = [
'dto' => $result,
'localPart' => $result->localPart,
'source_field' => $result->sourceField,
'source_keyword' => $result->sourceKeyword
];
}
}
return $uniqueResults;
}
public function prefetchDetailsForResults(array $searchResults): void
{
$prefetchCount = 0;
foreach ($searchResults as $partResult) {
foreach ($partResult['search_results'] as $result) {
$dto = $result['dto'];
try {
$this->infoRetriever->getDetails($dto->provider_key, $dto->provider_id);
$prefetchCount++;
} catch (\Exception $e) {
$this->logger->warning('Failed to prefetch details for provider part', [
'provider_key' => $dto->provider_key,
'provider_id' => $dto->provider_id,
'error' => $e->getMessage()
]);
}
}
}
$this->logger->info("Prefetched details for {$prefetchCount} search results");
}
}

14
src/Services/InfoProviderSystem/DTOs/BulkSearchRequestDTO.php Normal file
View file

@ -0,0 +1,14 @@
<?php
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
class BulkSearchRequestDTO
{
public function __construct(
public readonly array $fieldMappings,
public readonly bool $prefetchDetails = false,
public readonly array $partIds = []
) {}
}

32
src/Services/InfoProviderSystem/DTOs/BulkSearchResultDTO.php Normal file
View file

@ -0,0 +1,32 @@
<?php
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
class BulkSearchResultDTO extends SearchResultDTO
{
public function __construct(
SearchResultDTO $baseDto,
public readonly ?string $sourceField = null,
public readonly ?string $sourceKeyword = null,
public readonly ?Part $localPart = null,
public readonly int $priority = 1
) {
parent::__construct(
provider_key: $baseDto->provider_key,
provider_id: $baseDto->provider_id,
name: $baseDto->name,
description: $baseDto->description,
category: $baseDto->category,
manufacturer: $baseDto->manufacturer,
mpn: $baseDto->mpn,
preview_image_url: $baseDto->preview_image_url,
manufacturing_status: $baseDto->manufacturing_status,
provider_url: $baseDto->provider_url,
footprint: $baseDto->footprint
);
}
}