groove/Part-DB-server
1
0
Fork 0
mirror of https://github.com/Part-DB/Part-DB-server.git synced 2026-05-10 15:12:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Added URL delegation feature to AI provider and added option to skip that delegation

Browse source
This commit is contained in:
Jan Böhmer 2026-05-02 23:42:26 +02:00
parent aac5b8e0be
commit 889aa08b4e
9 changed files with 121 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/Controller/InfoProviderController.php
View file

@ -240,12 +240,16 @@ class InfoProviderController extends AbstractController
$method = $form->get('method')->getData();
$no_cache = $form->get('no_cache')->getData();
$skip_delegation = $form->get('skip_delegation')->getData();
try {
//It's okay if we use the cached results here, as its just for convenience
$searchResult = $this->infoRetriever->searchByKeyword(
keyword: $url,
providers: [$method],
options: [
InfoProviderInterface::OPTION_SKIP_DELEGATION => $skip_delegation,
]
);
if (count($searchResult) === 0) {
@ -257,6 +261,7 @@ class InfoProviderController extends AbstractController
'providerKey' => $searchResult->provider_key,
'providerId' => $searchResult->provider_id,
'no_cache' => $no_cache ? 1 : null,
'skip_delegation' => $skip_delegation ? 1 : null,
]);
}
} catch (ExceptionInterface $e) {

6
src/Controller/PartController.php
View file

@ -286,8 +286,12 @@ final class PartController extends AbstractController
//Force info providers to not use cache, when retrieving part details for creating a new part, because otherwise we might end up with outdated information
$no_cache = $request->query->getBoolean('no_cache', false);
$skip_delegation = $request->query->getBoolean('skip_delegation', false);
$dto = $infoRetriever->getDetails($providerKey, $providerId, [InfoProviderInterface::OPTION_NO_CACHE => $no_cache]);
$dto = $infoRetriever->getDetails($providerKey, $providerId, [
InfoProviderInterface::OPTION_NO_CACHE => $no_cache,
InfoProviderInterface::OPTION_SKIP_DELEGATION => $skip_delegation,
]);
$new_part = $infoRetriever->dtoToPart($dto);
if ($new_part->getCategory() === null || $new_part->getCategory()->getID() === null) {

5
src/Form/InfoProviderSystem/FromURLFormType.php
View file

@ -75,6 +75,11 @@ class FromURLFormType extends AbstractType
'required' => false,
]);
$builder->add('skip_delegation', CheckboxType::class, [
'label' => 'info_providers.from_url.skip_delegation',
'required' => false,
]);
$builder->add('submit', SubmitType::class, [
'label' => 'info_providers.search.submit',
]);

59
src/Services/InfoProviderSystem/CreateFromUrlHelper.php
View file

@ -24,11 +24,18 @@ declare(strict_types=1);
namespace App\Services\InfoProviderSystem;
use App\Entity\UserSystem\User;
use App\Exceptions\ProviderIDNotSupportedException;
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
use Symfony\Bundle\SecurityBundle\Security;
final readonly class CreateFromUrlHelper
{
public function __construct(private Security $security, private ProviderRegistry $providerRegistry)
public function __construct(private Security $security,
private ProviderRegistry $providerRegistry,
private PartInfoRetriever $infoRetriever,
)
{
}
@ -49,4 +56,54 @@ final readonly class CreateFromUrlHelper
return $genericWebProvider->isActive() || $aiWebProvider->isActive();
}
/**
* Delegates the URL to another provider if possible, otherwise return null
* @param string $url
* @return SearchResultDTO|null
*/
public function delegateToOtherProvider(string $url, InfoProviderInterface $callingInfoProvider): ?SearchResultDTO
{
//Extract domain from url:
$host = parse_url($url, PHP_URL_HOST);
if ($host === false || $host === null) {
return null;
}
$provider = $this->providerRegistry->getProviderHandlingDomain($host);
if ($provider !== null && $provider->isActive() && $provider->getProviderKey() !== $callingInfoProvider->getProviderKey()) {
try {
$id = $provider->getIDFromURL($url);
if ($id !== null) {
$results = $this->infoRetriever->searchByKeyword($id, [$provider]);
if (count($results) > 0) {
return $results[0];
}
}
return null;
} catch (ProviderIDNotSupportedException $e) {
//Ignore and continue
return null;
}
}
return null;
}
/**
* Delegates the URL to another provider if possible and returns the details, otherwise return null
* @param string $url
* @param InfoProviderInterface $callingInfoProvider
* @return PartDetailDTO|null
*/
public function delegateToOtherProviderDetails(string $url, InfoProviderInterface $callingInfoProvider): ?PartDetailDTO
{
$delegatedResult = $this->delegateToOtherProvider($url, $callingInfoProvider);
if ($delegatedResult !== null) {
return $this->infoRetriever->getDetailsForSearchResult($delegatedResult);
}
return null;
}
}

28
src/Services/InfoProviderSystem/Providers/AIWebProvider.php
View file

@ -27,6 +27,7 @@ namespace App\Services\InfoProviderSystem\Providers;
use App\Exceptions\ProviderIDNotSupportedException;
use App\Helpers\RandomizeUseragentHttpClient;
use App\Services\AI\AIPlatformRegistry;
use App\Services\InfoProviderSystem\CreateFromUrlHelper;
use App\Services\InfoProviderSystem\DTOJsonSchemaConverter;
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
use App\Settings\InfoProviderSystem\AIExtractorSettings;
@ -57,7 +58,8 @@ final class AIWebProvider implements InfoProviderInterface
private readonly AIExtractorSettings $settings,
private readonly AIPlatformRegistry $AIPlatformRegistry,
private readonly DTOJsonSchemaConverter $jsonSchemaConverter,
private readonly CacheItemPoolInterface $partInfoCache
private readonly CacheItemPoolInterface $partInfoCache,
private readonly CreateFromUrlHelper $createFromUrlHelper,
) {
//Use NoPrivateNetworkHttpClient to prevent SSRF vulnerabilities, and RandomizeUseragentHttpClient to make it harder for servers to block us
$this->httpClient = (new RandomizeUseragentHttpClient(new NoPrivateNetworkHttpClient($httpClient)))->withOptions(
@ -90,9 +92,23 @@ final class AIWebProvider implements InfoProviderInterface
public function searchByKeyword(string $keyword, array $options = []): array
{
$url = $this->fixAndValidateURL($keyword);
if (!($options[self::OPTION_SKIP_DELEGATION] ?? false)) {
//Before loading the page, try to delegate to another provider
$delegatedPart = $this->createFromUrlHelper->delegateToOtherProvider($url, $this);
if ($delegatedPart !== null) {
return [$delegatedPart];
}
}
try {
$new_options = $options;
$new_options[self::OPTION_SKIP_DELEGATION] = true; //Skip delegation for the getDetails call to prevent infinite loops
return [
$this->getDetails($keyword, $options)
$this->getDetails($keyword, $new_options)
]; } catch (ProviderIDNotSupportedException $e) {
return [];
}
@ -102,6 +118,14 @@ final class AIWebProvider implements InfoProviderInterface
{
$url = $this->fixAndValidateURL($id);
if (!($options[self::OPTION_SKIP_DELEGATION] ?? false)) {
//Before loading the page, try to delegate to another provider
$delegatedPart = $this->createFromUrlHelper->delegateToOtherProviderDetails($url, $this);
if ($delegatedPart !== null) {
return $delegatedPart;
}
}
//Check if we have a cached result for this URL, to avoid unnecessary LLM calls, which can be slow and costly.
$cacheKey = 'ai_web_'.hash('xxh3', $url);

62
src/Services/InfoProviderSystem/Providers/GenericWebProvider.php
View file

@ -25,6 +25,7 @@ namespace App\Services\InfoProviderSystem\Providers;
use App\Exceptions\ProviderIDNotSupportedException;
use App\Helpers\RandomizeUseragentHttpClient;
use App\Services\InfoProviderSystem\CreateFromUrlHelper;
use App\Services\InfoProviderSystem\DTOs\ParameterDTO;
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
use App\Services\InfoProviderSystem\DTOs\PriceDTO;
@ -50,14 +51,12 @@ class GenericWebProvider implements InfoProviderInterface
use FixAndValidateUrlTrait;
public const OPTION_CHECK_FOR_DELEGATION = 'check_for_delegation';
public const DISTRIBUTOR_NAME = 'Website';
private readonly HttpClientInterface $httpClient;
public function __construct(HttpClientInterface $httpClient, private readonly GenericWebProviderSettings $settings,
private readonly ProviderRegistry $providerRegistry, private readonly PartInfoRetriever $infoRetriever,
private readonly CreateFromUrlHelper $createFromUrlHelper,
)
{
//Use NoPrivateNetworkHttpClient to prevent SSRF vulnerabilities, and RandomizeUseragentHttpClient to make it harder for servers to block us
@ -93,15 +92,19 @@ class GenericWebProvider implements InfoProviderInterface
{
$url = $this->fixAndValidateURL($keyword);
//Before loading the page, try to delegate to another provider
$delegatedPart = $this->delegateToOtherProvider($url);
if ($delegatedPart !== null) {
return [$delegatedPart];
if (!($options[self::OPTION_SKIP_DELEGATION] ?? false)) {
//Before loading the page, try to delegate to another provider
$delegatedPart = $this->createFromUrlHelper->delegateToOtherProvider($url, $this);
if ($delegatedPart !== null) {
return [$delegatedPart];
}
}
try {
$new_options = $options;
$new_options[self::OPTION_SKIP_DELEGATION] = true; //Skip delegation for the getDetails call to prevent infinite loops
return [
$this->getDetails($keyword, [self::OPTION_CHECK_FOR_DELEGATION => false]) //We already tried delegation
$this->getDetails($keyword, $new_options)
]; } catch (ProviderIDNotSupportedException $e) {
return [];
}
@ -278,53 +281,16 @@ class GenericWebProvider implements InfoProviderInterface
return null;
}
/**
* Delegates the URL to another provider if possible, otherwise return null
* @param string $url
* @return SearchResultDTO|null
*/
private function delegateToOtherProvider(string $url): ?SearchResultDTO
{
//Extract domain from url:
$host = parse_url($url, PHP_URL_HOST);
if ($host === false || $host === null) {
return null;
}
$provider = $this->providerRegistry->getProviderHandlingDomain($host);
if ($provider !== null && $provider->isActive() && $provider->getProviderKey() !== $this->getProviderKey()) {
try {
$id = $provider->getIDFromURL($url);
if ($id !== null) {
$results = $this->infoRetriever->searchByKeyword($id, [$provider]);
if (count($results) > 0) {
return $results[0];
}
}
return null;
} catch (ProviderIDNotSupportedException $e) {
//Ignore and continue
return null;
}
}
return null;
}
public function getDetails(string $id, array $options = []): PartDetailDTO
{
//We check for delegation by default
$check_for_delegation = $options[self::OPTION_CHECK_FOR_DELEGATION] ?? true;
$url = $this->fixAndValidateURL($id);
if ($check_for_delegation) {
if (!($options[self::OPTION_SKIP_DELEGATION] ?? false)) {
//Before loading the page, try to delegate to another provider
$delegatedPart = $this->delegateToOtherProvider($url);
$delegatedPart = $this->createFromUrlHelper->delegateToOtherProviderDetails($url, $this);
if ($delegatedPart !== null) {
return $this->infoRetriever->getDetailsForSearchResult($delegatedPart);
return $delegatedPart;
}
}

1
src/Services/InfoProviderSystem/Providers/InfoProviderInterface.php
View file

@ -29,6 +29,7 @@ use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
interface InfoProviderInterface
{
public const OPTION_NO_CACHE = 'no_cache'; // if set to true, the provider should not use any cache and retrieve fresh data from the source
public const OPTION_SKIP_DELEGATION = 'skip_delegation'; // if set to true, the provider should not delegate the request to other providers, even if it supports delegation.
/**
* Get information about this provider

1
templates/info_providers/from_url/from_url.html.twig
View file

@ -28,6 +28,7 @@
<div class="collapse" id="infoSearchAdvancedPanel">
<div class="card card-body mb-2">
{{ form_row(form.no_cache) }}
{{ form_row(form.skip_delegation) }}
</div>
</div>

6
translations/messages.en.xlf
View file

@ -13181,5 +13181,11 @@ Buerklin-API Authentication server:
<target>Ignore cache / Force fresh info retrieval</target>
</segment>
</unit>
<unit id="302Jgvm" name="info_providers.from_url.skip_delegation">
<segment>
<source>info_providers.from_url.skip_delegation</source>
<target>Do not delegate to specialized info providers</target>
</segment>
</unit>
</file>
</xliff>