From 6ac0a8a678bdfd702e53cabd9a09cfcf44aabc82 Mon Sep 17 00:00:00 2001 From: Cutch Date: Mon, 24 Nov 2025 22:46:57 -0500 Subject: [PATCH 1/6] Add id to the metadata json and use it when moving files if ino cannot be matched --- server/models/LibraryItem.js | 2 ++ server/scanner/BookScanner.js | 1 + server/scanner/LibraryScanner.js | 14 +++++++++++++- server/scanner/PodcastScanner.js | 1 + 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/server/models/LibraryItem.js b/server/models/LibraryItem.js index 16a521615..3051aacc8 100644 --- a/server/models/LibraryItem.js +++ b/server/models/LibraryItem.js @@ -574,6 +574,7 @@ class LibraryItem extends Model { let jsonObject = {} if (this.mediaType === 'book') { jsonObject = { + absId: this.id, tags: mediaExpanded.tags || [], chapters: mediaExpanded.chapters?.map((c) => ({ ...c })) || [], title: mediaExpanded.title, @@ -598,6 +599,7 @@ class LibraryItem extends Model { } } else { jsonObject = { + absId: this.id, tags: mediaExpanded.tags || [], title: mediaExpanded.title, author: mediaExpanded.author, diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js index a1e7ff507..fa3695fed 100644 --- a/server/scanner/BookScanner.js +++ b/server/scanner/BookScanner.js @@ -821,6 +821,7 @@ class BookScanner { const metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`) const jsonObject = { + absId: libraryItem.id, tags: libraryItem.media.tags || [], chapters: libraryItem.media.chapters?.map((c) => ({ ...c })) || [], title: libraryItem.media.title, diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 640c82d76..79cc194f1 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -14,6 +14,7 @@ const LibraryItemScanner = require('./LibraryItemScanner') const LibraryScan = require('./LibraryScan') const LibraryItemScanData = require('./LibraryItemScanData') const Task = require('../objects/Task') +const abmetadataGenerator = require('../utils/generators/abmetadataGenerator') class LibraryScanner { constructor() { @@ -574,7 +575,7 @@ class LibraryScanner { let updatedLibraryItemDetails = {} if (!existingLibraryItem) { const isSingleMedia = isSingleMediaFile(fileUpdateGroup, itemDir) - existingLibraryItem = (await findLibraryItemByItemToItemInoMatch(library.id, fullPath)) || (await findLibraryItemByItemToFileInoMatch(library.id, fullPath, isSingleMedia)) || (await findLibraryItemByFileToItemInoMatch(library.id, fullPath, isSingleMedia, fileUpdateGroup[itemDir])) + existingLibraryItem = (await findLibraryItemByItemToItemInoMatch(library.id, fullPath)) || (await findLibraryItemByItemToFileInoMatch(library.id, fullPath, isSingleMedia)) || (await findLibraryItemByFileToItemInoMatch(library.id, fullPath, isSingleMedia, fileUpdateGroup[itemDir])) || (await findLibraryItemByItemToMetadata(fullPath, isSingleMedia)) if (existingLibraryItem) { // Update library item paths for scan existingLibraryItem.path = fullPath @@ -709,3 +710,14 @@ async function findLibraryItemByFileToItemInoMatch(libraryId, fullPath, isSingle if (existingLibraryItem) Logger.debug(`[LibraryScanner] Found library item with inode matching one of "${itemFileInos.join(',')}" at path "${existingLibraryItem.path}"`) return existingLibraryItem } + +async function findLibraryItemByItemToMetadata(fullPath, isSingleMedia) { + if (isSingleMedia) return null + const metadataText = await fileUtils.readTextFile(Path.join(fullPath, 'metadata.json')) + if (!metadataText) return null + const abMetadata = abmetadataGenerator.parseJson(metadataText) || {} + // check if metadata id exists in the database + const existingLibraryItem = await Database.libraryItemModel.getExpandedById(abMetadata.absId) + if (existingLibraryItem) Logger.debug(`[LibraryScanner] Found library item with metadata id matching one of "${abMetadata.absId}" at path "${existingLibraryItem.path}"`) + return existingLibraryItem +} diff --git a/server/scanner/PodcastScanner.js b/server/scanner/PodcastScanner.js index c9569c3ad..d2ccf334e 100644 --- a/server/scanner/PodcastScanner.js +++ b/server/scanner/PodcastScanner.js @@ -421,6 +421,7 @@ class PodcastScanner { const metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`) const jsonObject = { + absId: libraryItem.id, tags: libraryItem.media.tags || [], title: libraryItem.media.title, author: libraryItem.media.author, From 736a939284c2e20c10c113698549cf7af384b527 Mon Sep 17 00:00:00 2001 From: Cutch Date: Mon, 24 Nov 2025 22:48:12 -0500 Subject: [PATCH 2/6] Add api to update all metadata files --- server/controllers/ToolsController.js | 21 +++++++++++++++++++++ server/routers/ApiRouter.js | 1 + 2 files changed, 22 insertions(+) diff --git a/server/controllers/ToolsController.js b/server/controllers/ToolsController.js index 9f2014ec0..767d5e467 100644 --- a/server/controllers/ToolsController.js +++ b/server/controllers/ToolsController.js @@ -150,6 +150,27 @@ class ToolsController { res.sendStatus(200) } + /** + * POST: /api/tools/batch/update-metadata-files + * Start batch request to update all metadata files + * + * @this import('../routers/ApiRouter') + * + * @param {RequestWithUser} req + * @param {Response} res + */ + async updateAllItemMetadata(req, res) { + if (!req.user.isAdminOrUp) { + Logger.warn(`Non-admin user "${req.user.username}" other than admin attempted to batch scan library items`) + return res.sendStatus(403) + } + + const libraryItems = await Database.libraryItemModel.findAll() + for (const libraryItem of libraryItems) await libraryItem.saveMetadataFile() + + res.sendStatus(200) + } + /** * * @param {RequestWithUser} req diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index db04bf5ec..2ddbcf424 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -298,6 +298,7 @@ class ApiRouter { this.router.delete('/tools/item/:id/encode-m4b', ToolsController.middleware.bind(this), ToolsController.cancelM4bEncode.bind(this)) this.router.post('/tools/item/:id/embed-metadata', ToolsController.middleware.bind(this), ToolsController.embedAudioFileMetadata.bind(this)) this.router.post('/tools/batch/embed-metadata', ToolsController.middleware.bind(this), ToolsController.batchEmbedMetadata.bind(this)) + this.router.post('/tools/batch/update-metadata-files', ToolsController.middleware.bind(this), ToolsController.updateAllItemMetadata.bind(this)) // // RSS Feed Routes (Admin and up) From 63a97b3c40b7cc67ac49d75fa0ad9f207d501db7 Mon Sep 17 00:00:00 2001 From: Cutch Date: Mon, 24 Nov 2025 23:06:32 -0500 Subject: [PATCH 3/6] Add check for copied files --- server/scanner/LibraryScanner.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 79cc194f1..142a73a2e 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -718,6 +718,15 @@ async function findLibraryItemByItemToMetadata(fullPath, isSingleMedia) { const abMetadata = abmetadataGenerator.parseJson(metadataText) || {} // check if metadata id exists in the database const existingLibraryItem = await Database.libraryItemModel.getExpandedById(abMetadata.absId) - if (existingLibraryItem) Logger.debug(`[LibraryScanner] Found library item with metadata id matching one of "${abMetadata.absId}" at path "${existingLibraryItem.path}"`) + + if (existingLibraryItem) { + Logger.debug(`[LibraryScanner] Found library item with metadata id matching one of "${abMetadata.absId}" at path "${existingLibraryItem.path}"`) + + for (const { metadata } of existingLibraryItem.getLibraryFiles()) + if (await fs.pathExists(metadata.path)) { + Logger.debug(`[LibraryScanner] Conflicting library files exist "${metadata.path}"`) + return null + } + } return existingLibraryItem } From d5cd4f794469b8b0ef5a6956b36cfe52ca4825c7 Mon Sep 17 00:00:00 2001 From: Cutch Date: Mon, 24 Nov 2025 23:26:06 -0500 Subject: [PATCH 4/6] Fix debug log metadata save title --- server/controllers/ToolsController.js | 4 +++- server/models/LibraryItem.js | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/controllers/ToolsController.js b/server/controllers/ToolsController.js index 767d5e467..385310575 100644 --- a/server/controllers/ToolsController.js +++ b/server/controllers/ToolsController.js @@ -166,7 +166,9 @@ class ToolsController { } const libraryItems = await Database.libraryItemModel.findAll() - for (const libraryItem of libraryItems) await libraryItem.saveMetadataFile() + for (const libraryItem of libraryItems) { + await libraryItem.saveMetadataFile() + } res.sendStatus(200) } diff --git a/server/models/LibraryItem.js b/server/models/LibraryItem.js index 3051aacc8..32f62b9a5 100644 --- a/server/models/LibraryItem.js +++ b/server/models/LibraryItem.js @@ -649,7 +649,7 @@ class LibraryItem extends Model { } } - Logger.debug(`[LibraryItem] Saved metadata for "${this.media.title}" file to "${metadataFilePath}"`) + Logger.debug(`[LibraryItem] Saved metadata for "${mediaExpanded.title}" file to "${metadataFilePath}"`) return metadataLibraryFile }) From 94e13014ed27cd2023f0b792d8b26fb7457034cd Mon Sep 17 00:00:00 2001 From: Cutch Date: Tue, 25 Nov 2025 09:21:26 -0500 Subject: [PATCH 5/6] Update absId to id --- server/models/LibraryItem.js | 4 ++-- server/scanner/BookScanner.js | 2 +- server/scanner/LibraryScanner.js | 4 ++-- server/scanner/PodcastScanner.js | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/models/LibraryItem.js b/server/models/LibraryItem.js index 32f62b9a5..d2a22586a 100644 --- a/server/models/LibraryItem.js +++ b/server/models/LibraryItem.js @@ -574,7 +574,7 @@ class LibraryItem extends Model { let jsonObject = {} if (this.mediaType === 'book') { jsonObject = { - absId: this.id, + id: this.id, tags: mediaExpanded.tags || [], chapters: mediaExpanded.chapters?.map((c) => ({ ...c })) || [], title: mediaExpanded.title, @@ -599,7 +599,7 @@ class LibraryItem extends Model { } } else { jsonObject = { - absId: this.id, + id: this.id, tags: mediaExpanded.tags || [], title: mediaExpanded.title, author: mediaExpanded.author, diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js index fa3695fed..7e639b474 100644 --- a/server/scanner/BookScanner.js +++ b/server/scanner/BookScanner.js @@ -821,7 +821,7 @@ class BookScanner { const metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`) const jsonObject = { - absId: libraryItem.id, + id: libraryItem.id, tags: libraryItem.media.tags || [], chapters: libraryItem.media.chapters?.map((c) => ({ ...c })) || [], title: libraryItem.media.title, diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 142a73a2e..64c9aae96 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -717,10 +717,10 @@ async function findLibraryItemByItemToMetadata(fullPath, isSingleMedia) { if (!metadataText) return null const abMetadata = abmetadataGenerator.parseJson(metadataText) || {} // check if metadata id exists in the database - const existingLibraryItem = await Database.libraryItemModel.getExpandedById(abMetadata.absId) + const existingLibraryItem = await Database.libraryItemModel.getExpandedById(abMetadata.id) if (existingLibraryItem) { - Logger.debug(`[LibraryScanner] Found library item with metadata id matching one of "${abMetadata.absId}" at path "${existingLibraryItem.path}"`) + Logger.debug(`[LibraryScanner] Found library item with metadata id matching one of "${abMetadata.id}" at path "${existingLibraryItem.path}"`) for (const { metadata } of existingLibraryItem.getLibraryFiles()) if (await fs.pathExists(metadata.path)) { diff --git a/server/scanner/PodcastScanner.js b/server/scanner/PodcastScanner.js index d2ccf334e..090ab9f77 100644 --- a/server/scanner/PodcastScanner.js +++ b/server/scanner/PodcastScanner.js @@ -421,7 +421,7 @@ class PodcastScanner { const metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`) const jsonObject = { - absId: libraryItem.id, + id: libraryItem.id, tags: libraryItem.media.tags || [], title: libraryItem.media.title, author: libraryItem.media.author, From dbcad610cb12ebdff71564c4b6fe03b465745ae3 Mon Sep 17 00:00:00 2001 From: Cutch Date: Tue, 2 Dec 2025 18:36:33 -0500 Subject: [PATCH 6/6] Add migration --- server/migrations/changelog.md | 1 + .../v2.31.1-update-metadata-json-with-id.js | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 server/migrations/v2.31.1-update-metadata-json-with-id.js diff --git a/server/migrations/changelog.md b/server/migrations/changelog.md index 0fcbe6754..4f826725d 100644 --- a/server/migrations/changelog.md +++ b/server/migrations/changelog.md @@ -16,3 +16,4 @@ Please add a record of every database migration that you create to this file. Th | v2.19.1 | v2.19.1-copy-title-to-library-items | Copies title and titleIgnorePrefix to the libraryItems table, creates update triggers and indices | | v2.19.4 | v2.19.4-improve-podcast-queries | Adds numEpisodes to podcasts, adds podcastId to mediaProgresses, copies podcast title to libraryItems | | v2.20.0 | v2.20.0-improve-author-sort-queries | Adds AuthorNames(FirstLast\|LastFirst) to libraryItems to improve author sort queries | +| v2.31.1 | v2.31.1-update-metadata-json-with-id | Adds ids to the locally stored metadata.json to help file moves keep track of the items | diff --git a/server/migrations/v2.31.1-update-metadata-json-with-id.js b/server/migrations/v2.31.1-update-metadata-json-with-id.js new file mode 100644 index 000000000..4e3ae780b --- /dev/null +++ b/server/migrations/v2.31.1-update-metadata-json-with-id.js @@ -0,0 +1,48 @@ +/** + * @typedef MigrationContext + * @property {import('sequelize').QueryInterface} queryInterface - a sequelize QueryInterface object. + * @property {import('../Logger')} logger - a Logger object. + * + * @typedef MigrationOptions + * @property {MigrationContext} context - an object containing the migration context. + */ + +const migrationVersion = '2.31.1' +const migrationName = `${migrationVersion}-update-metadata-json-with-id` +const loggerPrefix = `[${migrationVersion} migration]` + +/** + * This upward migration creates a sessions table and apiKeys table. + * + * @param {MigrationOptions} options - an object containing the migration context. + * @returns {Promise} - A promise that resolves when the migration is complete. + */ +async function up({ context: { logger } }) { + // Upwards migration script + logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) + // Re-save all metadata json files, the id field will be added + const libraryItems = await Database.libraryItemModel.findAll() + for (const libraryItem of libraryItems) { + await libraryItem.saveMetadataFile() + } + logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`) +} + +/** + * This downward migration script removes the sessions table and apiKeys table. + * + * @param {MigrationOptions} options - an object containing the migration context. + * @returns {Promise} - A promise that resolves when the migration is complete. + */ +async function down({ context: { logger } }) { + // Downward migration script + logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) + // Re-save all metadata json files, the id field will be removed + const libraryItems = await Database.libraryItemModel.findAll() + for (const libraryItem of libraryItems) { + await libraryItem.saveMetadataFile() + } + logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`) +} + +module.exports = { up, down }