groove/Part-DB-server
1
0
Fork 0
mirror of https://github.com/Part-DB/Part-DB-server.git synced 2026-01-13 13:49:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Merge branch 'feature/batch-info-provider-import'

Browse source
This commit is contained in:
Jan Böhmer 2025-09-21 23:14:09 +02:00
commit ed1e51f694
80 changed files with 9789 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

110
src/Services/ImportExportSystem/EntityExporter.php
View file

@ -38,6 +38,9 @@ use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
use Symfony\Component\Serializer\SerializerInterface;
use function Symfony\Component\String\u;
use PhpOffice\PhpSpreadsheet\Spreadsheet;
use PhpOffice\PhpSpreadsheet\Writer\Xlsx;
use PhpOffice\PhpSpreadsheet\Writer\Xls;
/**
* Use this class to export an entity to multiple file formats.
@ -52,7 +55,7 @@ class EntityExporter
protected function configureOptions(OptionsResolver $resolver): void
{
$resolver->setDefault('format', 'csv');
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml']);
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml', 'xlsx', 'xls']);
$resolver->setDefault('csv_delimiter', ';');
$resolver->setAllowedTypes('csv_delimiter', 'string');
@ -88,28 +91,35 @@ class EntityExporter
$options = $resolver->resolve($options);
//Handle Excel formats by converting from CSV
if (in_array($options['format'], ['xlsx', 'xls'], true)) {
return $this->exportToExcel($entities, $options);
}
//If include children is set, then we need to add the include_children group
$groups = [$options['level']];
if ($options['include_children']) {
$groups[] = 'include_children';
}
return $this->serializer->serialize($entities, $options['format'],
return $this->serializer->serialize(
$entities,
$options['format'],
[
'groups' => $groups,
'as_collection' => true,
'csv_delimiter' => $options['csv_delimiter'],
'xml_root_node_name' => 'PartDBExport',
'partdb_export' => true,
//Skip the item normalizer, so that we dont get IRIs in the output
//Skip the item normalizer, so that we dont get IRIs in the output
SkippableItemNormalizer::DISABLE_ITEM_NORMALIZER => true,
//Handle circular references
//Handle circular references
AbstractNormalizer::CIRCULAR_REFERENCE_HANDLER => $this->handleCircularReference(...),
]
);
}
private function handleCircularReference(object $object, string $format, array $context): string
private function handleCircularReference(object $object): string
{
if ($object instanceof AbstractStructuralDBElement) {
return $object->getFullPath("->");
@ -119,7 +129,75 @@ class EntityExporter
return $object->__toString();
}
throw new CircularReferenceException('Circular reference detected for object of type '.get_class($object));
throw new CircularReferenceException('Circular reference detected for object of type ' . get_class($object));
}
/**
* Exports entities to Excel format (xlsx or xls).
*
* @param AbstractNamedDBElement[] $entities The entities to export
* @param array $options The export options
*
* @return string The Excel file content as binary string
*/
protected function exportToExcel(array $entities, array $options): string
{
//First get CSV data using existing serializer
$groups = [$options['level']];
if ($options['include_children']) {
$groups[] = 'include_children';
}
$csvData = $this->serializer->serialize(
$entities,
'csv',
[
'groups' => $groups,
'as_collection' => true,
'csv_delimiter' => $options['csv_delimiter'],
'partdb_export' => true,
SkippableItemNormalizer::DISABLE_ITEM_NORMALIZER => true,
AbstractNormalizer::CIRCULAR_REFERENCE_HANDLER => $this->handleCircularReference(...),
]
);
//Convert CSV to Excel
$spreadsheet = new Spreadsheet();
$worksheet = $spreadsheet->getActiveSheet();
$rows = explode("\n", $csvData);
$rowIndex = 1;
foreach ($rows as $row) {
if (trim($row) === '') {
continue;
}
$columns = str_getcsv($row, $options['csv_delimiter'], '"', '\\');
$colIndex = 1;
foreach ($columns as $column) {
$cellCoordinate = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($colIndex) . $rowIndex;
$worksheet->setCellValue($cellCoordinate, $column);
$colIndex++;
}
$rowIndex++;
}
//Save to memory stream
$writer = $options['format'] === 'xlsx' ? new Xlsx($spreadsheet) : new Xls($spreadsheet);
$memFile = fopen("php://temp", 'r+b');
$writer->save($memFile);
rewind($memFile);
$content = stream_get_contents($memFile);
fclose($memFile);
if ($content === false) {
throw new \RuntimeException('Failed to read Excel content from memory stream.');
}
return $content;
}
/**
@ -156,19 +234,15 @@ class EntityExporter
//Determine the content type for the response
//Plain text should work for all types
$content_type = 'text/plain';
//Try to use better content types based on the format
$format = $options['format'];
switch ($format) {
case 'xml':
$content_type = 'application/xml';
break;
case 'json':
$content_type = 'application/json';
break;
}
$content_type = match ($format) {
'xml' => 'application/xml',
'json' => 'application/json',
'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'xls' => 'application/vnd.ms-excel',
default => 'text/plain',
};
$response->headers->set('Content-Type', $content_type);
//If view option is not specified, then download the file.
@ -186,7 +260,7 @@ class EntityExporter
$level = $options['level'];
$filename = 'export_'.$entity_name.'_'.$level.'.'.$format;
$filename = "export_{$entity_name}_{$level}.{$format}";
//Sanitize the filename
$filename = FilenameSanatizer::sanitizeFilename($filename);

139
src/Services/ImportExportSystem/EntityImporter.php
View file

@ -38,6 +38,9 @@ use Symfony\Component\HttpFoundation\File\File;
use Symfony\Component\OptionsResolver\OptionsResolver;
use Symfony\Component\Serializer\SerializerInterface;
use Symfony\Component\Validator\Validator\ValidatorInterface;
use PhpOffice\PhpSpreadsheet\IOFactory;
use PhpOffice\PhpSpreadsheet\Spreadsheet;
use Psr\Log\LoggerInterface;
/**
* @see \App\Tests\Services\ImportExportSystem\EntityImporterTest
@ -50,7 +53,7 @@ class EntityImporter
*/
private const ENCODINGS = ["ASCII", "UTF-8", "ISO-8859-1", "ISO-8859-15", "Windows-1252", "UTF-16", "UTF-32"];
public function __construct(protected SerializerInterface $serializer, protected EntityManagerInterface $em, protected ValidatorInterface $validator)
public function __construct(protected SerializerInterface $serializer, protected EntityManagerInterface $em, protected ValidatorInterface $validator, protected LoggerInterface $logger)
{
}
@ -102,7 +105,7 @@ class EntityImporter
foreach ($names as $name) {
//Count indentation level (whitespace characters at the beginning of the line)
$identSize = strlen($name)-strlen(ltrim($name));
$identSize = strlen($name) - strlen(ltrim($name));
//If the line is intended more than the last line, we have a new parent element
if ($identSize > end($indentations)) {
@ -195,16 +198,20 @@ class EntityImporter
}
//The [] behind class_name denotes that we expect an array.
$entities = $this->serializer->deserialize($data, $options['class'].'[]', $options['format'],
$entities = $this->serializer->deserialize(
$data,
$options['class'] . '[]',
$options['format'],
[
'groups' => $groups,
'csv_delimiter' => $options['csv_delimiter'],
'create_unknown_datastructures' => $options['create_unknown_datastructures'],
'path_delimiter' => $options['path_delimiter'],
'partdb_import' => true,
//Disable API Platform normalizer, as we don't want to use it here
//Disable API Platform normalizer, as we don't want to use it here
SkippableItemNormalizer::DISABLE_ITEM_NORMALIZER => true,
]);
]
);
//Ensure we have an array of entity elements.
if (!is_array($entities)) {
@ -279,7 +286,7 @@ class EntityImporter
'path_delimiter' => '->', //The delimiter used to separate the path elements in the name of a structural element
]);
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml']);
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml', 'xlsx', 'xls']);
$resolver->setAllowedTypes('csv_delimiter', 'string');
$resolver->setAllowedTypes('preserve_children', 'bool');
$resolver->setAllowedTypes('class', 'string');
@ -335,6 +342,33 @@ class EntityImporter
*/
public function importFile(File $file, array $options = [], array &$errors = []): array
{
$resolver = new OptionsResolver();
$this->configureOptions($resolver);
$options = $resolver->resolve($options);
if (in_array($options['format'], ['xlsx', 'xls'], true)) {
$this->logger->info('Converting Excel file to CSV', [
'filename' => $file->getFilename(),
'format' => $options['format'],
'delimiter' => $options['csv_delimiter']
]);
$csvData = $this->convertExcelToCsv($file, $options['csv_delimiter']);
$options['format'] = 'csv';
$this->logger->debug('Excel to CSV conversion completed', [
'csv_length' => strlen($csvData),
'csv_lines' => substr_count($csvData, "\n") + 1
]);
// Log the converted CSV for debugging (first 1000 characters)
$this->logger->debug('Converted CSV preview', [
'csv_preview' => substr($csvData, 0, 1000) . (strlen($csvData) > 1000 ? '...' : '')
]);
return $this->importString($csvData, $options, $errors);
}
return $this->importString($file->getContent(), $options, $errors);
}
@ -354,10 +388,103 @@ class EntityImporter
'xml' => 'xml',
'csv', 'tsv' => 'csv',
'yaml', 'yml' => 'yaml',
'xlsx' => 'xlsx',
'xls' => 'xls',
default => null,
};
}
/**
* Converts Excel file to CSV format using PhpSpreadsheet.
*
* @param File $file The Excel file to convert
* @param string $delimiter The CSV delimiter to use
*
* @return string The CSV data as string
*/
protected function convertExcelToCsv(File $file, string $delimiter = ';'): string
{
try {
$this->logger->debug('Loading Excel file', ['path' => $file->getPathname()]);
$spreadsheet = IOFactory::load($file->getPathname());
$worksheet = $spreadsheet->getActiveSheet();
$csvData = [];
$highestRow = $worksheet->getHighestRow();
$highestColumn = $worksheet->getHighestColumn();
$this->logger->debug('Excel file dimensions', [
'rows' => $highestRow,
'columns_detected' => $highestColumn,
'worksheet_title' => $worksheet->getTitle()
]);
$highestColumnIndex = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($highestColumn);
for ($row = 1; $row <= $highestRow; $row++) {
$rowData = [];
// Read all columns using numeric index
for ($colIndex = 1; $colIndex <= $highestColumnIndex; $colIndex++) {
$col = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($colIndex);
try {
$cellValue = $worksheet->getCell("{$col}{$row}")->getCalculatedValue();
$rowData[] = $cellValue ?? '';
} catch (\Exception $e) {
$this->logger->warning('Error reading cell value', [
'cell' => "{$col}{$row}",
'error' => $e->getMessage()
]);
$rowData[] = '';
}
}
$csvRow = implode($delimiter, array_map(function ($value) use ($delimiter) {
$value = (string) $value;
if (strpos($value, $delimiter) !== false || strpos($value, '"') !== false || strpos($value, "\n") !== false) {
return '"' . str_replace('"', '""', $value) . '"';
}
return $value;
}, $rowData));
$csvData[] = $csvRow;
// Log first few rows for debugging
if ($row <= 3) {
$this->logger->debug("Row {$row} converted", [
'original_data' => $rowData,
'csv_row' => $csvRow,
'first_cell_raw' => $worksheet->getCell("A{$row}")->getValue(),
'first_cell_calculated' => $worksheet->getCell("A{$row}")->getCalculatedValue()
]);
}
}
$result = implode("\n", $csvData);
$this->logger->info('Excel to CSV conversion successful', [
'total_rows' => count($csvData),
'total_characters' => strlen($result)
]);
$this->logger->debug('Full CSV data', [
'csv_data' => $result
]);
return $result;
} catch (\Exception $e) {
$this->logger->error('Failed to convert Excel to CSV', [
'file' => $file->getFilename(),
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString()
]);
throw $e;
}
}
/**
* This functions corrects the parent setting based on the children value of the parent.
*