groove/Part-DB-server
1
0
Fork 0
mirror of https://github.com/Part-DB/Part-DB-server.git synced 2026-01-14 14:19:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge tag 'v2.2.1' into Buerklin-provider

Browse source
This commit is contained in:
Marc Kreidler 2025-12-06 15:10:53 +01:00
commit 7de735eb1e
191 changed files with 27939 additions and 2310 deletions
Download patch file Download diff file Expand all files Collapse all files

40
src/Services/InfoProviderSystem/Providers/BatchInfoProviderInterface.php Normal file
View file

@ -0,0 +1,40 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\Providers;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
/**
* This interface marks a provider as a info provider which can provide information directly in batch operations
*/
interface BatchInfoProviderInterface extends InfoProviderInterface
{
/**
* Search for multiple keywords in a single batch operation and return the results, ordered by the keywords.
* This allows for a more efficient search compared to running multiple single searches.
* @param string[] $keywords
* @return array<string, SearchResultDTO[]> An associative array where the key is the keyword and the value is the search results for that keyword
*/
public function searchByKeywordsBatch(array $keywords): array;
}

36
src/Services/InfoProviderSystem/Providers/DigikeyProvider.php
View file

@ -24,6 +24,7 @@ declare(strict_types=1);
namespace App\Services\InfoProviderSystem\Providers;
use App\Entity\Parts\ManufacturingStatus;
use App\Exceptions\OAuthReconnectRequiredException;
use App\Services\InfoProviderSystem\DTOs\FileDTO;
use App\Services\InfoProviderSystem\DTOs\ParameterDTO;
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
@ -117,12 +118,22 @@ class DigikeyProvider implements InfoProviderInterface
];
//$response = $this->digikeyClient->request('POST', '/Search/v3/Products/Keyword', [
$response = $this->digikeyClient->request('POST', '/products/v4/search/keyword', [
'json' => $request,
'auth_bearer' => $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME)
]);
try {
$response = $this->digikeyClient->request('POST', '/products/v4/search/keyword', [
'json' => $request,
'auth_bearer' => $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME)
]);
$response_array = $response->toArray();
} catch (\InvalidArgumentException $exception) {
//Check if the exception was caused by an invalid or expired token
if (str_contains($exception->getMessage(), 'access_token')) {
throw OAuthReconnectRequiredException::forProvider($this->getProviderKey());
}
throw $exception;
}
$response_array = $response->toArray();
$result = [];
@ -150,9 +161,18 @@ class DigikeyProvider implements InfoProviderInterface
public function getDetails(string $id): PartDetailDTO
{
$response = $this->digikeyClient->request('GET', '/products/v4/search/' . urlencode($id) . '/productdetails', [
'auth_bearer' => $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME)
]);
try {
$response = $this->digikeyClient->request('GET', '/products/v4/search/' . urlencode($id) . '/productdetails', [
'auth_bearer' => $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME)
]);
} catch (\InvalidArgumentException $exception) {
//Check if the exception was caused by an invalid or expired token
if (str_contains($exception->getMessage(), 'access_token')) {
throw OAuthReconnectRequiredException::forProvider($this->getProviderKey());
}
throw $exception;
}
$response_array = $response->toArray();
$product = $response_array['Product'];

76
src/Services/InfoProviderSystem/Providers/EmptyProvider.php Normal file
View file

@ -0,0 +1,76 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\Providers;
use App\Services\InfoProviderSystem\DTOs\FileDTO;
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
use Symfony\Component\DependencyInjection\Attribute\When;
/**
* This is a provider, which is used during tests. It always returns no results.
*/
#[When(env: 'test')]
class EmptyProvider implements InfoProviderInterface
{
public function getProviderInfo(): array
{
return [
'name' => 'Empty Provider',
'description' => 'This is a test provider',
//'url' => 'https://example.com',
'disabled_help' => 'This provider is disabled for testing purposes'
];
}
public function getProviderKey(): string
{
return 'empty';
}
public function isActive(): bool
{
return true;
}
public function searchByKeyword(string $keyword): array
{
return [
];
}
public function getCapabilities(): array
{
return [
ProviderCapabilities::BASIC,
ProviderCapabilities::FOOTPRINT,
];
}
public function getDetails(string $id): PartDetailDTO
{
throw new \RuntimeException('No part details available');
}
}

145
src/Services/InfoProviderSystem/Providers/LCSCProvider.php
View file

@ -33,7 +33,7 @@ use App\Settings\InfoProviderSystem\LCSCSettings;
use Symfony\Component\HttpFoundation\Cookie;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class LCSCProvider implements InfoProviderInterface
class LCSCProvider implements BatchInfoProviderInterface
{
private const ENDPOINT_URL = 'https://wmsc.lcsc.com/ftps/wm';
@ -69,9 +69,10 @@ class LCSCProvider implements InfoProviderInterface
/**
* @param string $id
* @param bool $lightweight If true, skip expensive operations like datasheet resolution
* @return PartDetailDTO
*/
private function queryDetail(string $id): PartDetailDTO
private function queryDetail(string $id, bool $lightweight = false): PartDetailDTO
{
$response = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/product/detail", [
'headers' => [
@ -89,7 +90,7 @@ class LCSCProvider implements InfoProviderInterface
throw new \RuntimeException('Could not find product code: ' . $id);
}
return $this->getPartDetail($product);
return $this->getPartDetail($product, $lightweight);
}
/**
@ -99,30 +100,42 @@ class LCSCProvider implements InfoProviderInterface
private function getRealDatasheetUrl(?string $url): string
{
if ($url !== null && trim($url) !== '' && preg_match("/^https:\/\/(datasheet\.lcsc\.com|www\.lcsc\.com\/datasheet)\/.*(C\d+)\.pdf$/", $url, $matches) > 0) {
if (preg_match("/^https:\/\/datasheet\.lcsc\.com\/lcsc\/(.*\.pdf)$/", $url, $rewriteMatches) > 0) {
$url = 'https://www.lcsc.com/datasheet/lcsc_datasheet_' . $rewriteMatches[1];
}
$response = $this->lcscClient->request('GET', $url, [
'headers' => [
'Referer' => 'https://www.lcsc.com/product-detail/_' . $matches[2] . '.html'
],
]);
if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.lcsc\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) {
//HACKY: The URL string contains escaped characters like \u002F, etc. To decode it, the JSON decoding is reused
//See https://github.com/Part-DB/Part-DB-server/pull/582#issuecomment-2033125934
$jsonObj = json_decode('{"' . $matches[1] . '": ' . $matches[2] . '}');
$url = $jsonObj->previewPdfUrl;
}
if (preg_match("/^https:\/\/datasheet\.lcsc\.com\/lcsc\/(.*\.pdf)$/", $url, $rewriteMatches) > 0) {
$url = 'https://www.lcsc.com/datasheet/lcsc_datasheet_' . $rewriteMatches[1];
}
$response = $this->lcscClient->request('GET', $url, [
'headers' => [
'Referer' => 'https://www.lcsc.com/product-detail/_' . $matches[2] . '.html'
],
]);
if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.lcsc\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) {
//HACKY: The URL string contains escaped characters like \u002F, etc. To decode it, the JSON decoding is reused
//See https://github.com/Part-DB/Part-DB-server/pull/582#issuecomment-2033125934
$jsonObj = json_decode('{"' . $matches[1] . '": ' . $matches[2] . '}');
$url = $jsonObj->previewPdfUrl;
}
}
return $url;
}
/**
* @param string $term
* @param bool $lightweight If true, skip expensive operations like datasheet resolution
* @return PartDetailDTO[]
*/
private function queryByTerm(string $term): array
private function queryByTerm(string $term, bool $lightweight = false): array
{
// Optimize: If term looks like an LCSC part number (starts with C followed by digits),
// use direct detail query instead of slower search
if (preg_match('/^C\d+$/i', trim($term))) {
try {
return [$this->queryDetail(trim($term), $lightweight)];
} catch (\Exception $e) {
// If direct lookup fails, fall back to search
// This handles cases where the C-code might not exist
}
}
$response = $this->lcscClient->request('POST', self::ENDPOINT_URL . "/search/v2/global", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
@ -145,11 +158,11 @@ class LCSCProvider implements InfoProviderInterface
// detailed product listing. It does so utilizing a product tip field.
// If product tip exists and there are no products in the product list try a detail query
if (count($products) === 0 && $tipProductCode !== null) {
$result[] = $this->queryDetail($tipProductCode);
$result[] = $this->queryDetail($tipProductCode, $lightweight);
}
foreach ($products as $product) {
$result[] = $this->getPartDetail($product);
$result[] = $this->getPartDetail($product, $lightweight);
}
return $result;
@ -178,7 +191,7 @@ class LCSCProvider implements InfoProviderInterface
* @param array $product
* @return PartDetailDTO
*/
private function getPartDetail(array $product): PartDetailDTO
private function getPartDetail(array $product, bool $lightweight = false): PartDetailDTO
{
// Get product images in advance
$product_images = $this->getProductImages($product['productImages'] ?? null);
@ -214,10 +227,10 @@ class LCSCProvider implements InfoProviderInterface
manufacturing_status: null,
provider_url: $this->getProductShortURL($product['productCode']),
footprint: $this->sanitizeField($footprint),
datasheets: $this->getProductDatasheets($product['pdfUrl'] ?? null),
images: $product_images,
parameters: $this->attributesToParameters($product['paramVOList'] ?? []),
vendor_infos: $this->pricesToVendorInfo($product['productCode'], $this->getProductShortURL($product['productCode']), $product['productPriceList'] ?? []),
datasheets: $lightweight ? [] : $this->getProductDatasheets($product['pdfUrl'] ?? null),
images: $product_images, // Always include images - users need to see them
parameters: $lightweight ? [] : $this->attributesToParameters($product['paramVOList'] ?? []),
vendor_infos: $lightweight ? [] : $this->pricesToVendorInfo($product['productCode'], $this->getProductShortURL($product['productCode']), $product['productPriceList'] ?? []),
mass: $product['weight'] ?? null,
);
}
@ -286,7 +299,7 @@ class LCSCProvider implements InfoProviderInterface
*/
private function getProductShortURL(string $product_code): string
{
return 'https://www.lcsc.com/product-detail/' . $product_code .'.html';
return 'https://www.lcsc.com/product-detail/' . $product_code . '.html';
}
/**
@ -327,7 +340,7 @@ class LCSCProvider implements InfoProviderInterface
//Skip this attribute if it's empty
if (in_array(trim((string) $attribute['paramValueEn']), ['', '-'], true)) {
continue;
continue;
}
$result[] = ParameterDTO::parseValueIncludingUnit(name: $attribute['paramNameEn'], value: $attribute['paramValueEn'], group: null);
@ -338,12 +351,86 @@ class LCSCProvider implements InfoProviderInterface
public function searchByKeyword(string $keyword): array
{
return $this->queryByTerm($keyword);
return $this->queryByTerm($keyword, true); // Use lightweight mode for search
}
/**
* Batch search multiple keywords asynchronously (like JavaScript Promise.all)
* @param array $keywords Array of keywords to search
* @return array Results indexed by keyword
*/
public function searchByKeywordsBatch(array $keywords): array
{
if (empty($keywords)) {
return [];
}
$responses = [];
$results = [];
// Start all requests immediately (like JavaScript promises without await)
foreach ($keywords as $keyword) {
if (preg_match('/^C\d+$/i', trim($keyword))) {
// Direct detail API call for C-codes
$responses[$keyword] = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/product/detail", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
],
'query' => [
'productCode' => trim($keyword),
],
]);
} else {
// Search API call for other terms
$responses[$keyword] = $this->lcscClient->request('POST', self::ENDPOINT_URL . "/search/v2/global", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
],
'json' => [
'keyword' => $keyword,
],
]);
}
}
// Now collect all results (like .then() in JavaScript)
foreach ($responses as $keyword => $response) {
try {
$arr = $response->toArray(); // This waits for the response
$results[$keyword] = $this->processSearchResponse($arr, $keyword);
} catch (\Exception $e) {
$results[$keyword] = []; // Empty results on error
}
}
return $results;
}
private function processSearchResponse(array $arr, string $keyword): array
{
$result = [];
// Check if this looks like a detail response (direct C-code lookup)
if (isset($arr['result']['productCode'])) {
$product = $arr['result'];
$result[] = $this->getPartDetail($product, true); // lightweight mode
} else {
// This is a search response
$products = $arr['result']['productSearchResultVO']['productList'] ?? [];
$tipProductCode = $arr['result']['tipProductDetailUrlVO']['productCode'] ?? null;
// If no products but has tip, we'd need another API call - skip for batch mode
foreach ($products as $product) {
$result[] = $this->getPartDetail($product, true); // lightweight mode
}
}
return $result;
}
public function getDetails(string $id): PartDetailDTO
{
$tmp = $this->queryByTerm($id);
$tmp = $this->queryByTerm($id, false);
if (count($tmp) === 0) {
throw new \RuntimeException('No part found with ID ' . $id);
}

32
src/Services/InfoProviderSystem/Providers/MouserProvider.php
View file

@ -132,6 +132,15 @@ class MouserProvider implements InfoProviderInterface
],
]);
// Check for API errors before processing response
if ($response->getStatusCode() !== 200) {
throw new \RuntimeException(sprintf(
'Mouser API returned HTTP %d: %s',
$response->getStatusCode(),
$response->getContent(false)
));
}
return $this->responseToDTOArray($response);
}
@ -169,6 +178,16 @@ class MouserProvider implements InfoProviderInterface
]
],
]);
// Check for API errors before processing response
if ($response->getStatusCode() !== 200) {
throw new \RuntimeException(sprintf(
'Mouser API returned HTTP %d: %s',
$response->getStatusCode(),
$response->getContent(false)
));
}
$tmp = $this->responseToDTOArray($response);
//Ensure that we have exactly one result
@ -286,6 +305,17 @@ class MouserProvider implements InfoProviderInterface
return (float)$val;
}
private function mapCurrencyCode(string $currency): string
{
//Mouser uses "RMB" for Chinese Yuan, but the correct ISO code is "CNY"
if ($currency === "RMB") {
return "CNY";
}
//For all other currencies, we assume that the ISO code is correct
return $currency;
}
/**
* Converts the pricing (StandardPricing field) from the Mouser API to an array of PurchaseInfoDTOs
* @param array $price_breaks
@ -302,7 +332,7 @@ class MouserProvider implements InfoProviderInterface
$prices[] = new PriceDTO(
minimum_discount_amount: $price_break['Quantity'],
price: (string)$number,
currency_iso_code: $price_break['Currency']
currency_iso_code: $this->mapCurrencyCode($price_break['Currency'])
);
}