From 29db029d69282d0eb23144bb5b1741cf1f478c3a Mon Sep 17 00:00:00 2001 From: Wieland Schopohl <55855374+wschopohl@users.noreply.github.com> Date: Wed, 15 Apr 2026 22:56:34 +0200 Subject: [PATCH] Add SI-prefix-aware sorting column for parts tableFeature/si value sort (#1344) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add SI-prefix-aware sorting column for the parts table Adds an optional "Name (SI)" column that parses numeric values with SI prefixes (p, n, u/µ, m, k/K, M, G, T) from part names and sorts by the resulting physical value. This is useful for electronic components where alphabetical sorting produces wrong results — e.g. 100nF, 10pF, 1uF should sort as 10pF < 100nF < 1uF. Implementation: - New SiValueSort DQL function with platform-specific SQL generation for PostgreSQL (POSIX regex), MySQL/MariaDB (REGEXP_SUBSTR), and SQLite (PHP callback registered via the existing middleware). - The regex is start-anchored: only names beginning with a number are matched. Part numbers like "MCP2515" or "Crystal 20MHz" are ignored. - When SI sort is active, NATSORT is appended as a secondary sort so that non-matching parts fall back to natural string ordering instead of appearing in arbitrary order. - The column is opt-in (not in default columns) and displays the parsed float value, or an empty cell for non-matching names. * Rename SI column from "Name (SI)" to "SI Value" The column now shows the parsed numeric value rather than the part name, so the label should reflect that. * Support comma as decimal separator in SI value parsing Part names using European decimal notation (e.g. "4,7 kΩ", "2,2uF") were parsed incorrectly because the regex only recognized dots. Now commas are normalized to dots before parsing, matching the existing pattern used elsewhere in the codebase (PartNormalizer, price providers). --- config/packages/doctrine.yaml | 1 + src/DataTables/PartsDataTable.php | 25 +++ src/Doctrine/Functions/SiValueSort.php | 196 ++++++++++++++++++ .../SQLiteRegexExtensionMiddlewareDriver.php | 4 + .../BehaviorSettings/PartTableColumns.php | 2 + tests/Doctrine/Functions/SiValueSortTest.php | 193 +++++++++++++++++ translations/messages.en.xlf | 6 + 7 files changed, 427 insertions(+) create mode 100644 src/Doctrine/Functions/SiValueSort.php create mode 100644 tests/Doctrine/Functions/SiValueSortTest.php diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index 5261c295..164ac717 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -56,6 +56,7 @@ doctrine: natsort: App\Doctrine\Functions\Natsort array_position: App\Doctrine\Functions\ArrayPosition ilike: App\Doctrine\Functions\ILike + si_value_sort: App\Doctrine\Functions\SiValueSort when@test: doctrine: diff --git a/src/DataTables/PartsDataTable.php b/src/DataTables/PartsDataTable.php index 8bb5f6aa..ca2abd45 100644 --- a/src/DataTables/PartsDataTable.php +++ b/src/DataTables/PartsDataTable.php @@ -38,6 +38,7 @@ use App\DataTables\Filters\PartFilter; use App\DataTables\Filters\PartSearchFilter; use App\DataTables\Helpers\ColumnSortHelper; use App\DataTables\Helpers\PartDataTableHelper; +use App\Doctrine\Functions\SiValueSort; use App\Doctrine\Helpers\FieldHelper; use App\Entity\Parts\ManufacturingStatus; use App\Entity\Parts\Part; @@ -118,6 +119,17 @@ final class PartsDataTable implements DataTableTypeInterface 'render' => fn($value, Part $context) => $this->partDataTableHelper->renderName($context), 'orderField' => 'NATSORT(part.name)' ]) + ->add('si_value', TextColumn::class, [ + 'label' => $this->translator->trans('part.table.si_value'), + 'render' => function ($value, Part $context): string { + $siValue = SiValueSort::sqliteSiValue($context->getName()); + if ($siValue !== null) { + return htmlspecialchars(sprintf('%g', $siValue)); + } + return ''; + }, + 'orderField' => 'SI_VALUE_SORT(part.name)', + ]) ->add('id', TextColumn::class, [ 'label' => $this->translator->trans('part.table.id'), ]) @@ -484,6 +496,19 @@ final class PartsDataTable implements DataTableTypeInterface //$builder->addGroupBy('_bulkImportJob'); } + //When sorting by SI value, add NATSORT as a secondary sort so that parts without + //an SI-prefixed value fall back to natural string ordering seamlessly. + $orderByParts = $builder->getDQLPart('orderBy'); + foreach ($orderByParts as $orderBy) { + foreach ($orderBy->getParts() as $part) { + if (str_contains($part, 'SI_VALUE_SORT')) { + $direction = str_contains($part, 'DESC') ? 'DESC' : 'ASC'; + $builder->addOrderBy('NATSORT(part.name)', $direction); + break 2; + } + } + } + return $builder; } diff --git a/src/Doctrine/Functions/SiValueSort.php b/src/Doctrine/Functions/SiValueSort.php new file mode 100644 index 00000000..1bba1b9f --- /dev/null +++ b/src/Doctrine/Functions/SiValueSort.php @@ -0,0 +1,196 @@ +. + */ + +declare(strict_types=1); + +namespace App\Doctrine\Functions; + +use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; +use Doctrine\DBAL\Platforms\SQLitePlatform; +use Doctrine\ORM\Query\AST\Functions\FunctionNode; +use Doctrine\ORM\Query\AST\Node; +use Doctrine\ORM\Query\Parser; +use Doctrine\ORM\Query\SqlWalker; +use Doctrine\ORM\Query\TokenType; + +/** + * Custom DQL function that extracts the first numeric value with an optional SI prefix + * from a string and returns the scaled numeric value for sorting. + * + * Usage: SI_VALUE_SORT(part.name) + * + * This enables sorting parts by their physical value. For example, capacitors + * named "100nF", "1uF", "10pF" will be sorted by actual value: 10pF < 100nF < 1uF. + * + * Supported SI prefixes: p (pico, 1e-12), n (nano, 1e-9), u/µ (micro, 1e-6), + * m (milli, 1e-3), k/K (kilo, 1e3), M (mega, 1e6), G (giga, 1e9), T (tera, 1e12). + * + * Only matches numbers at the very beginning of the string (ignoring leading whitespace). + * Names like "Crystal 20MHz" will NOT match since the number is not at the start. + * Names without a recognizable numeric+prefix pattern return NULL and sort last. + */ +class SiValueSort extends FunctionNode +{ + private ?Node $field = null; + + /** + * SI prefix multipliers. Used by the SQLite PHP callback. + */ + private const SI_MULTIPLIERS = [ + 'p' => 1e-12, + 'n' => 1e-9, + 'u' => 1e-6, + 'µ' => 1e-6, + 'm' => 1e-3, + 'k' => 1e3, + 'K' => 1e3, + 'M' => 1e6, + 'G' => 1e9, + 'T' => 1e12, + ]; + + public function parse(Parser $parser): void + { + $parser->match(TokenType::T_IDENTIFIER); + $parser->match(TokenType::T_OPEN_PARENTHESIS); + + $this->field = $parser->ArithmeticExpression(); + + $parser->match(TokenType::T_CLOSE_PARENTHESIS); + } + + public function getSql(SqlWalker $sqlWalker): string + { + assert($this->field !== null, 'Field is not set'); + + $platform = $sqlWalker->getConnection()->getDatabasePlatform(); + $rawField = $this->field->dispatch($sqlWalker); + + // Normalize comma decimal separator to dot for SQL platforms (European locale support) + $fieldSql = "REPLACE({$rawField}, ',', '.')"; + + if ($platform instanceof PostgreSQLPlatform) { + return $this->getPostgreSQLSql($fieldSql); + } + + if ($platform instanceof AbstractMySQLPlatform) { + return $this->getMySQLSql($fieldSql); + } + + // SQLite: comma normalization is handled in the PHP callback + $fieldSql = $rawField; + + if ($platform instanceof SQLitePlatform) { + return "SI_VALUE({$fieldSql})"; + } + + // Fallback: return NULL (no SI sorting available) + return 'NULL'; + } + + /** + * PostgreSQL implementation using substring() with POSIX regex. + */ + private function getPostgreSQLSql(string $field): string + { + // Extract the numeric part using POSIX regex, anchored at start (with optional leading whitespace) + $numericPart = "CAST(substring({$field} FROM '^\\s*(\\d+\\.?\\d*)\\s*[pnuµmkKMGT]?') AS DOUBLE PRECISION)"; + + // Extract the SI prefix character + $prefixPart = "substring({$field} FROM '^\\s*\\d+\\.?\\d*\\s*([pnuµmkKMGT])')"; + + return $this->buildCaseExpression($numericPart, $prefixPart); + } + + /** + * MySQL/MariaDB implementation using REGEXP_SUBSTR. + */ + private function getMySQLSql(string $field): string + { + // Extract the numeric part, anchored at start (with optional leading whitespace) + $numericPart = "CAST(REGEXP_SUBSTR({$field}, '^[[:space:]]*[0-9]+\\.?[0-9]*') AS DECIMAL(30,15))"; + + // Extract the prefix: get the full number+prefix match anchored at start, then take the last char + $fullMatch = "REGEXP_SUBSTR({$field}, '^[[:space:]]*[0-9]+\\.?[0-9]*[[:space:]]*[pnuµmkKMGT]')"; + $prefixPart = "RIGHT({$fullMatch}, 1)"; + + return $this->buildCaseExpression($numericPart, $prefixPart); + } + + /** + * Build a CASE expression that maps an SI prefix character to a multiplier + * and multiplies it with the numeric value. + * + * @param string $numericExpr SQL expression that evaluates to the numeric part + * @param string $prefixExpr SQL expression that evaluates to the SI prefix character + * @return string SQL CASE expression + */ + private function buildCaseExpression(string $numericExpr, string $prefixExpr): string + { + return "(CASE" . + " WHEN {$numericExpr} IS NULL THEN NULL" . + " WHEN {$prefixExpr} = 'p' THEN {$numericExpr} * 1e-12" . + " WHEN {$prefixExpr} = 'n' THEN {$numericExpr} * 1e-9" . + " WHEN {$prefixExpr} = 'u' THEN {$numericExpr} * 1e-6" . + " WHEN {$prefixExpr} = 'µ' THEN {$numericExpr} * 1e-6" . + " WHEN {$prefixExpr} = 'm' THEN {$numericExpr} * 1e-3" . + " WHEN {$prefixExpr} = 'k' THEN {$numericExpr} * 1e3" . + " WHEN {$prefixExpr} = 'K' THEN {$numericExpr} * 1e3" . + " WHEN {$prefixExpr} = 'M' THEN {$numericExpr} * 1e6" . + " WHEN {$prefixExpr} = 'G' THEN {$numericExpr} * 1e9" . + " WHEN {$prefixExpr} = 'T' THEN {$numericExpr} * 1e12" . + " ELSE {$numericExpr} * 1" . + " END)"; + } + + /** + * PHP callback for SQLite's SI_VALUE function. + * Extracts the first numeric value with an optional SI prefix and returns the scaled value. + * + * @param string|null $value The input string + * @return float|null The scaled numeric value, or null if no number found + */ + public static function sqliteSiValue(?string $value): ?float + { + if ($value === null) { + return null; + } + + // Normalize comma decimal separator to dot (European locale support) + $value = str_replace(',', '.', $value); + + // Match a number at the very start (allowing leading whitespace), optionally followed by an SI prefix + if (!preg_match('/^\s*(\d+\.?\d*)\s*([pnuµmkKMGT])?/u', $value, $matches)) { + return null; + } + + $number = (float) $matches[1]; + $prefix = $matches[2] ?? ''; + + if ($prefix === '') { + return $number; + } + + $multiplier = self::SI_MULTIPLIERS[$prefix] ?? 1.0; + + return $number * $multiplier; + } +} diff --git a/src/Doctrine/Middleware/SQLiteRegexExtensionMiddlewareDriver.php b/src/Doctrine/Middleware/SQLiteRegexExtensionMiddlewareDriver.php index ad572d4c..aa6108c9 100644 --- a/src/Doctrine/Middleware/SQLiteRegexExtensionMiddlewareDriver.php +++ b/src/Doctrine/Middleware/SQLiteRegexExtensionMiddlewareDriver.php @@ -23,6 +23,7 @@ declare(strict_types=1); namespace App\Doctrine\Middleware; +use App\Doctrine\Functions\SiValueSort; use App\Exceptions\InvalidRegexException; use Doctrine\DBAL\Driver\Connection; use Doctrine\DBAL\Driver\Middleware\AbstractDriverMiddleware; @@ -51,6 +52,9 @@ class SQLiteRegexExtensionMiddlewareDriver extends AbstractDriverMiddleware //Create a new collation for natural sorting $native_connection->sqliteCreateCollation('NATURAL_CMP', strnatcmp(...)); + + //Create a function for SI prefix value sorting + $native_connection->sqliteCreateFunction('SI_VALUE', SiValueSort::sqliteSiValue(...), 1, \PDO::SQLITE_DETERMINISTIC); } } diff --git a/src/Settings/BehaviorSettings/PartTableColumns.php b/src/Settings/BehaviorSettings/PartTableColumns.php index 3b30e0a4..32f6100b 100644 --- a/src/Settings/BehaviorSettings/PartTableColumns.php +++ b/src/Settings/BehaviorSettings/PartTableColumns.php @@ -52,6 +52,8 @@ enum PartTableColumns : string implements TranslatableInterface case TAGS = "tags"; case ATTACHMENTS = "attachments"; + case SI_VALUE = "si_value"; + case EDA_REFERENCE = "eda_reference"; case EDA_VALUE = "eda_value"; diff --git a/tests/Doctrine/Functions/SiValueSortTest.php b/tests/Doctrine/Functions/SiValueSortTest.php new file mode 100644 index 00000000..dbdd9d28 --- /dev/null +++ b/tests/Doctrine/Functions/SiValueSortTest.php @@ -0,0 +1,193 @@ +. + */ + +declare(strict_types=1); + +namespace App\Tests\Doctrine\Functions; + +use App\Doctrine\Functions\SiValueSort; +use Doctrine\DBAL\Platforms\MySQLPlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; +use Doctrine\DBAL\Platforms\SQLitePlatform; + +final class SiValueSortTest extends AbstractDoctrineFunctionTestCase +{ + public function testPostgreSQLGeneratesCaseExpression(): void + { + $function = new SiValueSort('SI_VALUE_SORT'); + $this->setObjectProperty($function, 'field', $this->createNode('part_name')); + + $sql = $function->getSql($this->createSqlWalker(new PostgreSQLPlatform())); + + $this->assertStringContainsString('CASE', $sql); + $this->assertStringContainsString("REPLACE(part_name, ',', '.')", $sql); + $this->assertStringContainsString('1e-12', $sql); + $this->assertStringContainsString('1e-9', $sql); + $this->assertStringContainsString('1e-6', $sql); + $this->assertStringContainsString('1e-3', $sql); + $this->assertStringContainsString('1e3', $sql); + $this->assertStringContainsString('1e6', $sql); + $this->assertStringContainsString('1e9', $sql); + $this->assertStringContainsString('1e12', $sql); + } + + public function testMySQLGeneratesCaseExpression(): void + { + $function = new SiValueSort('SI_VALUE_SORT'); + $this->setObjectProperty($function, 'field', $this->createNode('part_name')); + + $sql = $function->getSql($this->createSqlWalker(new MySQLPlatform())); + + $this->assertStringContainsString('CASE', $sql); + $this->assertStringContainsString("REPLACE(part_name, ',', '.')", $sql); + $this->assertStringContainsString('1e-12', $sql); + $this->assertStringContainsString('1e6', $sql); + } + + public function testSQLiteUsesSiValueFunction(): void + { + $function = new SiValueSort('SI_VALUE_SORT'); + $this->setObjectProperty($function, 'field', $this->createNode('part_name')); + + $sql = $function->getSql($this->createSqlWalker(new SQLitePlatform())); + + $this->assertSame('SI_VALUE(part_name)', $sql); + } + + /** + * @dataProvider sqliteSiValueProvider + */ + public function testSqliteSiValue(?string $input, ?float $expected): void + { + $result = SiValueSort::sqliteSiValue($input); + + if ($expected === null) { + $this->assertNull($result); + } else { + $this->assertEqualsWithDelta($expected, $result, $expected * 1e-9); + } + } + + /** + * @return iterable + */ + public static function sqliteSiValueProvider(): iterable + { + // Basic SI prefix values + yield 'pico' => ['10pF', 10e-12]; + yield 'nano' => ['100nF', 100e-9]; + yield 'micro_u' => ['1uF', 1e-6]; + yield 'micro_µ' => ['1µF', 1e-6]; + yield 'milli' => ['4.7mH', 4.7e-3]; + yield 'kilo_lower' => ['4.7k', 4.7e3]; + yield 'kilo_upper' => ['4.7K', 4.7e3]; + yield 'mega' => ['1M', 1e6]; + yield 'giga' => ['2.2G', 2.2e9]; + yield 'tera' => ['1T', 1e12]; + + // No prefix (plain number) + yield 'plain_integer' => ['100', 100.0]; + yield 'plain_decimal' => ['4.7', 4.7]; + + // Decimal values with prefix (dot separator) + yield 'decimal_nano' => ['4.7nF', 4.7e-9]; + yield 'decimal_micro' => ['0.1uF', 0.1e-6]; + yield 'decimal_kilo' => ['2.2k', 2.2e3]; + + // Comma decimal separator (European locale) + yield 'comma_kilo' => ['4,7k', 4.7e3]; + yield 'comma_micro' => ['2,2uF', 2.2e-6]; + yield 'comma_kilo_space' => ['1,2 kΩ', 1.2e3]; + + // Number NOT at the start — should return NULL + yield 'prefixed_name' => ['CAP-100nF', null]; + yield 'name_with_number' => ['R 4.7k 1%', null]; + yield 'crystal' => ['Crystal 20MHz', null]; + + // Number at start with trailing text + yield 'number_with_suffix' => ['10nF 25V', 10e-9]; + + // Space between number and prefix + yield 'space_before_prefix' => ['100 nF', 100e-9]; + + // Leading whitespace before number + yield 'leading_whitespace' => [' 10uF', 10e-6]; + + // No number at all + yield 'no_number' => ['Connector', null]; + yield 'text_only' => ['LED red', null]; + + // Null input + yield 'null' => [null, null]; + + // Empty string + yield 'empty' => ['', null]; + } + + /** + * Test that the sort order is correct by comparing sqliteSiValue results. + */ + public function testSortOrder(): void + { + $parts = ['1uF', '100nF', '10pF', '10uF', '0.1mF', '1F', '10kF', '1MF']; + $expected = ['10pF', '100nF', '1uF', '10uF', '0.1mF', '1F', '10kF', '1MF']; + + // Sort using sqliteSiValue + usort($parts, static function (string $a, string $b): int { + $va = SiValueSort::sqliteSiValue($a); + $vb = SiValueSort::sqliteSiValue($b); + return $va <=> $vb; + }); + + $this->assertSame($expected, $parts); + } + + /** + * Test that NULL values sort last (after all numeric values). + */ + public function testNullSortsLast(): void + { + $parts = ['Connector', '100nF', 'LED red', '10pF']; + + usort($parts, static function (string $a, string $b): int { + $va = SiValueSort::sqliteSiValue($a); + $vb = SiValueSort::sqliteSiValue($b); + + // NULL sorts last + if ($va === null && $vb === null) { + return 0; + } + if ($va === null) { + return 1; + } + if ($vb === null) { + return -1; + } + + return $va <=> $vb; + }); + + $this->assertSame('10pF', $parts[0]); + $this->assertSame('100nF', $parts[1]); + // Last two should be the non-numeric names + $this->assertContains('Connector', array_slice($parts, 2)); + $this->assertContains('LED red', array_slice($parts, 2)); + } +} diff --git a/translations/messages.en.xlf b/translations/messages.en.xlf index 9e5ed157..4da88512 100644 --- a/translations/messages.en.xlf +++ b/translations/messages.en.xlf @@ -2780,6 +2780,12 @@ If you have done this incorrectly or if a computer is no longer trusted, you can Name + + + part.table.si_value + SI Value + + part.table.id