groove/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2026-05-17 00:41:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge 4e808e6770 into 47ea6b5092

Browse source
This commit is contained in:
Jason Axley 2026-05-06 13:51:20 +02:00 • committed by GitHub
commit f3a220c173
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 2441 additions and 304 deletions
Download patch file Download diff file Expand all files Collapse all files

8
server/Watcher.js
View file

@ -18,14 +18,14 @@ class FolderWatcher extends EventEmitter {
constructor() {
super()
/** @type {{id:string, name:string, libraryFolders:import('./models/Folder')[], paths:string[], watcher:Watcher[]}[]} */
/** @type {{id:string, name:string, libraryFolders:import('./models/LibraryFolder')[], paths:string[], watcher:Watcher[]}[]} */
this.libraryWatchers = []
/** @type {PendingFileUpdate[]} */
this.pendingFileUpdates = []
this.pendingDelay = 10000
/** @type {NodeJS.Timeout} */
/** @type {NodeJS.Timeout | null} */
this.pendingTimeout = null
/** @type {Task} */
/** @type {Task | null} */
this.pendingTask = null
this.filesBeingAdded = new Set()
@ -36,7 +36,7 @@ class FolderWatcher extends EventEmitter {
this.ignoreDirs = []
/** @type {string[]} */
this.pendingDirsToRemoveFromIgnore = []
/** @type {NodeJS.Timeout} */
/** @type {NodeJS.Timeout | null} */
this.removeFromIgnoreTimer = null
this.disabled = false

2
server/managers/LogManager.js
View file

@ -177,7 +177,7 @@ class LogManager {
* @returns {string}
*/
getMostRecentCurrentDailyLogs() {
return this.currentDailyLog?.logs.slice(-5000) || ''
return this.currentDailyLog?.logs.slice(-5000) || []
}
}
module.exports = LogManager

1
server/migrations/changelog.md
View file

@ -16,3 +16,4 @@ Please add a record of every database migration that you create to this file. Th
| v2.19.1 | v2.19.1-copy-title-to-library-items | Copies title and titleIgnorePrefix to the libraryItems table, creates update triggers and indices |
| v2.19.4 | v2.19.4-improve-podcast-queries | Adds numEpisodes to podcasts, adds podcastId to mediaProgresses, copies podcast title to libraryItems |
| v2.20.0 | v2.20.0-improve-author-sort-queries | Adds AuthorNames(FirstLast\|LastFirst) to libraryItems to improve author sort queries |
| v2.32.0 | v2.32.0-add-deviceId | Adds deviceId to libraryItems table to uniquely identify files in a filesystem |

193
server/migrations/v2.30.0-add-deviceId.js Normal file
View file

@ -0,0 +1,193 @@
const util = require('util')
const { Sequelize, DataTypes } = require('sequelize')
const fileUtils = require('../utils/fileUtils')
const LibraryItem = require('../models/LibraryItem')
/**
* @typedef MigrationContext
* @property {import('sequelize').QueryInterface} queryInterface - a sequelize QueryInterface object.
* @property {import('../Logger')} logger - a Logger object.
*
* @typedef MigrationOptions
* @property {MigrationContext} context - an object containing the migration context.
*/
const migrationVersion = '2.30.0'
const migrationName = `${migrationVersion}-add-deviceId`
const loggerPrefix = `[${migrationVersion} migration]`
// Migration constants
const libraryItemsTableName = 'libraryItems'
const columns = [{ name: 'deviceId', spec: { type: DataTypes.STRING, allowNull: true } }]
const columnNames = columns.map((column) => column.name).join(', ')
/**
* This upward migration adds a deviceId column to the libraryItems table and populates it.
* It also creates an index on the ino, deviceId columns.
*
* @param {MigrationOptions} options - an object containing the migration context.
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
*/
async function up({ context: { queryInterface, logger } }) {
const helper = new MigrationHelper(queryInterface, logger)
// Upwards migration script
logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
// Add authorNames columns to libraryItems table
await helper.addColumns()
// Populate authorNames columns with the author names for each libraryItem
// TODO
await helper.populateColumnsFromSource()
// Create indexes on the authorNames columns
await helper.addIndexes()
// Add index on ino and deviceId to the podcastEpisodes table
await helper.addIndex('libraryItems', ['ino', 'deviceId'])
logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
}
/**
* This downward migration removes a deviceId column to the libraryItems table, *
* It also removes the index on ino and deviceId from the libraryItems table.
*
* @param {MigrationOptions} options - an object containing the migration context.
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
*/
async function down({ context: { queryInterface, logger } }) {
// Downward migration script
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
const helper = new MigrationHelper(queryInterface, logger)
// Remove index on publishedAt from the podcastEpisodes table
await helper.removeIndex('libraryItems', ['ino', 'deviceId'])
// Remove indexes on the authorNames columns
await helper.removeIndexes()
// Remove authorNames columns from libraryItems table
await helper.removeColumns()
logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
}
class MigrationHelper {
constructor(queryInterface, logger) {
this.queryInterface = queryInterface
this.logger = logger
}
async addColumn(table, column, options) {
this.logger.info(`${loggerPrefix} adding column "${column}" to table "${table}"`)
const tableDescription = await this.queryInterface.describeTable(table)
if (!tableDescription[column]) {
await this.queryInterface.addColumn(table, column, options)
this.logger.info(`${loggerPrefix} added column "${column}" to table "${table}"`)
} else {
this.logger.info(`${loggerPrefix} column "${column}" already exists in table "${table}"`)
}
}
async addColumns() {
this.logger.info(`${loggerPrefix} adding ${columnNames} columns to ${libraryItemsTableName} table`)
for (const column of columns) {
await this.addColumn(libraryItemsTableName, column.name, column.spec)
}
this.logger.info(`${loggerPrefix} added ${columnNames} columns to ${libraryItemsTableName} table`)
}
async removeColumn(table, column) {
this.logger.info(`${loggerPrefix} removing column "${column}" from table "${table}"`)
const tableDescription = await this.queryInterface.describeTable(table)
if (tableDescription[column]) {
await this.queryInterface.sequelize.query(`ALTER TABLE ${table} DROP COLUMN ${column}`)
this.logger.info(`${loggerPrefix} removed column "${column}" from table "${table}"`)
} else {
this.logger.info(`${loggerPrefix} column "${column}" does not exist in table "${table}"`)
}
}
async removeColumns() {
this.logger.info(`${loggerPrefix} removing ${columnNames} columns from ${libraryItemsTableName} table`)
for (const column of columns) {
await this.removeColumn(libraryItemsTableName, column.name)
}
this.logger.info(`${loggerPrefix} removed ${columnNames} columns from ${libraryItemsTableName} table`)
}
// populate from existing files on filesystem
async populateColumnsFromSource() {
this.logger.info(`${loggerPrefix} populating ${columnNames} columns in ${libraryItemsTableName} table`)
// list all libraryItems
/** @type {[[LibraryItem], any]} */
const [libraryItems, metadata] = await this.queryInterface.sequelize.query('SELECT * FROM libraryItems')
// load file stats for all libraryItems
libraryItems.forEach(async (item) => {
const deviceId = await fileUtils.getDeviceId(item.path)
// set deviceId for each libraryItem
await this.queryInterface.sequelize.query(
`UPDATE :libraryItemsTableName
SET (deviceId) = (:deviceId)
WHERE id = :id`,
{
replacements: {
libraryItemsTableName: libraryItemsTableName,
deviceId: deviceId,
id: item.id
}
}
)
})
this.logger.info(`${loggerPrefix} populated ${columnNames} columns in ${libraryItems} table`)
}
async addIndex(tableName, columns) {
const columnString = columns.map((column) => util.inspect(column)).join(', ')
const indexName = convertToSnakeCase(`${tableName}_${columns.map((column) => (typeof column === 'string' ? column : column.name)).join('_')}`)
try {
this.logger.info(`${loggerPrefix} adding index on [${columnString}] to table ${tableName}. index name: ${indexName}"`)
await this.queryInterface.addIndex(tableName, columns)
this.logger.info(`${loggerPrefix} added index on [${columnString}] to table ${tableName}. index name: ${indexName}"`)
} catch (error) {
if (error.name === 'SequelizeDatabaseError' && error.message.includes('already exists')) {
this.logger.info(`${loggerPrefix} index [${columnString}] for table "${tableName}" already exists`)
} else {
throw error
}
}
}
async addIndexes() {
for (const column of columns) {
await this.addIndex(libraryItemsTableName, ['libraryId', 'mediaType', { name: column.name, collate: 'NOCASE' }])
}
}
async removeIndex(tableName, columns) {
this.logger.info(`${loggerPrefix} removing index [${columns.join(', ')}] from table "${tableName}"`)
await this.queryInterface.removeIndex(tableName, columns)
this.logger.info(`${loggerPrefix} removed index [${columns.join(', ')}] from table "${tableName}"`)
}
async removeIndexes() {
for (const column of columns) {
await this.removeIndex(libraryItemsTableName, ['libraryId', 'mediaType', column.name])
}
}
}
/**
* Utility function to convert a string to snake case, e.g. "titleIgnorePrefix" -> "title_ignore_prefix"
*
* @param {string} str - the string to convert to snake case.
* @returns {string} - the string in snake case.
*/
function convertToSnakeCase(str) {
return str.replace(/([A-Z])/g, '_$1').toLowerCase()
}
module.exports = { up, down, migrationName }

4
server/models/Book.js
View file

@ -9,10 +9,11 @@ const SocketAuthority = require('../SocketAuthority')
/**
* @typedef EBookFileObject
* @property {string} ino
* @property {string} deviceId
* @property {string} ebookFormat
* @property {number} addedAt
* @property {number} updatedAt
* @property {{filename:string, ext:string, path:string, relPath:strFing, size:number, mtimeMs:number, ctimeMs:number, birthtimeMs:number}} metadata
* @property {{filename:string, ext:string, path:string, relPath:string, size:number, mtimeMs:number, ctimeMs:number, birthtimeMs:number}} metadata
*/
/**
@ -46,6 +47,7 @@ const SocketAuthority = require('../SocketAuthority')
* @typedef AudioFileObject
* @property {number} index
* @property {string} ino
* @property {string} deviceId
* @property {{filename:string, ext:string, path:string, relPath:string, size:number, mtimeMs:number, ctimeMs:number, birthtimeMs:number}} metadata
* @property {number} addedAt
* @property {number} updatedAt

11
server/models/LibraryItem.js
View file

@ -11,6 +11,7 @@ const Podcast = require('./Podcast')
/**
* @typedef LibraryFileObject
* @property {string} ino
* @property {string} deviceId
* @property {boolean} isSupplementary
* @property {number} addedAt
* @property {number} updatedAt
@ -33,6 +34,8 @@ class LibraryItem extends Model {
/** @type {string} */
this.ino
/** @type {string} */
this.deviceId
/** @type {string} */
this.path
/** @type {string} */
this.relPath
@ -237,7 +240,7 @@ class LibraryItem extends Model {
* @param {import('sequelize').WhereOptions} where
* @param {import('sequelize').BindOrReplacements} [replacements]
* @param {import('sequelize').IncludeOptions} [include]
* @returns {Promise<LibraryItemExpanded>}
* @returns {Promise<LibraryItemExpanded | null>}
*/
static async findOneExpanded(where, replacements = null, include = null) {
const libraryItem = await this.findOne({
@ -289,7 +292,7 @@ class LibraryItem extends Model {
* @param {import('./Library')} library
* @param {import('./User')} user
* @param {object} options
* @returns {{ libraryItems:Object[], count:number }}
* @returns {Promise<{ libraryItems:Object[], count:number }>}
*/
static async getByFilterAndSort(library, user, options) {
let start = Date.now()
@ -728,6 +731,7 @@ class LibraryItem extends Model {
primaryKey: true
},
ino: DataTypes.STRING,
deviceId: DataTypes.STRING,
path: DataTypes.STRING,
relPath: DataTypes.STRING,
mediaId: DataTypes.UUID,
@ -752,6 +756,9 @@ class LibraryItem extends Model {
sequelize,
modelName: 'libraryItem',
indexes: [
{
fields: ['ino', 'deviceId']
},
{
fields: ['createdAt']
},

2
server/objects/Task.js
View file

@ -113,7 +113,7 @@ class Task {
/**
* Set task as finished
*
* @param {TaskString} [newDescriptionString] update description
* @param {TaskString | null} [newDescriptionString] update description
* @param {boolean} [clearDescription] clear description
*/
setFinished(newDescriptionString = null, clearDescription = false) {

17
server/objects/files/AudioFile.js
View file

@ -6,6 +6,7 @@ class AudioFile {
constructor(data) {
this.index = null
this.ino = null
this.deviceId = null
/** @type {FileMetadata} */
this.metadata = null
this.addedAt = null
@ -44,6 +45,7 @@ class AudioFile {
return {
index: this.index,
ino: this.ino,
deviceId: this.deviceId,
metadata: this.metadata.toJSON(),
addedAt: this.addedAt,
updatedAt: this.updatedAt,
@ -69,9 +71,13 @@ class AudioFile {
}
}
/**
* @param {{ index: any; ino: any; deviceId: any; metadata: any; addedAt: any; updatedAt: any; manuallyVerified: any; exclude: any; error: null; trackNumFromMeta: any; discNumFromMeta: any; trackNumFromFilename: any; cdNumFromFilename: undefined; discNumFromFilename: any; format: any; duration: any; bitRate: any; language: any; codec: null; timeBase: any; channels: any; channelLayout: any; chapters: any[]; embeddedCoverArt: null; metaTags: any; }} data
*/
construct(data) {
this.index = data.index
this.ino = data.ino
this.deviceId = data.deviceId
this.metadata = new FileMetadata(data.metadata || {})
this.addedAt = data.addedAt
this.updatedAt = data.updatedAt
@ -112,6 +118,7 @@ class AudioFile {
// New scanner creates AudioFile from AudioFileScanner
setDataFromProbe(libraryFile, probeData) {
this.ino = libraryFile.ino || null
this.deviceId = libraryFile.deviceId || null
if (libraryFile.metadata instanceof FileMetadata) {
this.metadata = libraryFile.metadata.clone()
@ -137,7 +144,7 @@ class AudioFile {
syncChapters(updatedChapters) {
if (this.chapters.length !== updatedChapters.length) {
this.chapters = updatedChapters.map(ch => ({ ...ch }))
this.chapters = updatedChapters.map((ch) => ({ ...ch }))
return true
} else if (updatedChapters.length === 0) {
if (this.chapters.length > 0) {
@ -154,7 +161,7 @@ class AudioFile {
}
}
if (hasUpdates) {
this.chapters = updatedChapters.map(ch => ({ ...ch }))
this.chapters = updatedChapters.map((ch) => ({ ...ch }))
}
return hasUpdates
}
@ -164,8 +171,8 @@ class AudioFile {
}
/**
*
* @param {AudioFile} scannedAudioFile
*
* @param {AudioFile} scannedAudioFile
* @returns {boolean} true if updates were made
*/
updateFromScan(scannedAudioFile) {
@ -196,4 +203,4 @@ class AudioFile {
return hasUpdated
}
}
module.exports = AudioFile
module.exports = AudioFile

12
server/objects/files/EBookFile.js
View file

@ -1,8 +1,12 @@
const FileMetadata = require('../metadata/FileMetadata')
class EBookFile {
/**
* @param {{ ino: any; deviceId: any; isSupplementary?: boolean; addedAt?: number; updatedAt?: number; metadata?: { filename: string; ext: string; path: string; relPath: string; size: number; mtimeMs: number; ctimeMs: number; birthtimeMs: number; }; libraryFolderId?: any; libraryId?: any; mediaType?: any; mtimeMs?: any; ctimeMs?: any; birthtimeMs?: any; path?: any; relPath?: any; isFile?: any; mediaMetadata?: any; libraryFiles?: any; }} file
*/
constructor(file) {
this.ino = null
this.deviceId = null
this.metadata = null
this.ebookFormat = null
this.addedAt = null
@ -13,8 +17,12 @@ class EBookFile {
}
}
/**
* @param {{ ino: any; deviceId: any; isSupplementary?: boolean | undefined; addedAt: any; updatedAt: any; metadata: any; libraryFolderId?: any; libraryId?: any; mediaType?: any; mtimeMs?: any; ctimeMs?: any; birthtimeMs?: any; path?: any; relPath?: any; isFile?: any; mediaMetadata?: any; libraryFiles?: any; ebookFormat?: any; }} file
*/
construct(file) {
this.ino = file.ino
this.deviceId = file.deviceId
this.metadata = new FileMetadata(file.metadata)
this.ebookFormat = file.ebookFormat || this.metadata.format
this.addedAt = file.addedAt
@ -24,6 +32,7 @@ class EBookFile {
toJSON() {
return {
ino: this.ino,
deviceId: this.deviceId,
metadata: this.metadata.toJSON(),
ebookFormat: this.ebookFormat,
addedAt: this.addedAt,
@ -37,6 +46,7 @@ class EBookFile {
setData(libraryFile) {
this.ino = libraryFile.ino
this.deviceId = libraryFile.deviceId
this.metadata = libraryFile.metadata.clone()
this.ebookFormat = libraryFile.metadata.format
this.addedAt = Date.now()
@ -58,4 +68,4 @@ class EBookFile {
return hasUpdated
}
}
module.exports = EBookFile
module.exports = EBookFile

29
server/objects/files/LibraryFile.js
View file

@ -1,11 +1,15 @@
const Path = require('path')
const { getFileTimestampsWithIno, filePathToPOSIX } = require('../../utils/fileUtils')
const fileUtils = require('../../utils/fileUtils')
const globals = require('../../utils/globals')
const FileMetadata = require('../metadata/FileMetadata')
class LibraryFile {
/**
* @param {{ ino: any; deviceId: any; metadata?: { filename: any; ext: any; path: any; relPath: any; size: any; mtimeMs: any; ctimeMs: any; birthtimeMs: any; } | { filename: string; ext: string; path: string; relPath: string; size: number; mtimeMs: number; ctimeMs: number; birthtimeMs: number; } | null; isSupplementary?: any; addedAt?: any; updatedAt?: any; fileType?: string; libraryFolderId?: any; libraryId?: any; mediaType?: any; mtimeMs?: any; ctimeMs?: any; birthtimeMs?: any; path?: any; relPath?: any; isFile?: any; mediaMetadata?: any; libraryFiles?: any; } | undefined} [file]
*/
constructor(file) {
this.ino = null
this.deviceId = null
this.metadata = null
this.isSupplementary = null
this.addedAt = null
@ -18,6 +22,7 @@ class LibraryFile {
construct(file) {
this.ino = file.ino
this.deviceId = file.deviceId
this.metadata = new FileMetadata(file.metadata)
this.isSupplementary = file.isSupplementary === undefined ? null : file.isSupplementary
this.addedAt = file.addedAt
@ -27,7 +32,8 @@ class LibraryFile {
toJSON() {
return {
ino: this.ino,
metadata: this.metadata.toJSON(),
deviceId: this.deviceId,
metadata: this.metadata ? this.metadata.toJSON() : null,
isSupplementary: this.isSupplementary,
addedAt: this.addedAt,
updatedAt: this.updatedAt,
@ -40,11 +46,13 @@ class LibraryFile {
}
get fileType() {
if (globals.SupportedImageTypes.includes(this.metadata.format)) return 'image'
if (globals.SupportedAudioTypes.includes(this.metadata.format)) return 'audio'
if (globals.SupportedEbookTypes.includes(this.metadata.format)) return 'ebook'
if (globals.TextFileTypes.includes(this.metadata.format)) return 'text'
if (globals.MetadataFileTypes.includes(this.metadata.format)) return 'metadata'
if (this.metadata) {
if (globals.SupportedImageTypes.includes(this.metadata.format)) return 'image'
if (globals.SupportedAudioTypes.includes(this.metadata.format)) return 'audio'
if (globals.SupportedEbookTypes.includes(this.metadata.format)) return 'ebook'
if (globals.TextFileTypes.includes(this.metadata.format)) return 'text'
if (globals.MetadataFileTypes.includes(this.metadata.format)) return 'metadata'
}
return 'unknown'
}
@ -61,14 +69,15 @@ class LibraryFile {
}
async setDataFromPath(path, relPath) {
var fileTsData = await getFileTimestampsWithIno(path)
var fileTsData = await fileUtils.getFileTimestampsWithIno(path)
var fileMetadata = new FileMetadata()
fileMetadata.setData(fileTsData)
fileMetadata.filename = Path.basename(relPath)
fileMetadata.path = filePathToPOSIX(path)
fileMetadata.relPath = filePathToPOSIX(relPath)
fileMetadata.path = fileUtils.filePathToPOSIX(path)
fileMetadata.relPath = fileUtils.filePathToPOSIX(relPath)
fileMetadata.ext = Path.extname(relPath)
this.ino = fileTsData.ino
this.deviceId = fileTsData.dev
this.metadata = fileMetadata
this.addedAt = Date.now()
this.updatedAt = Date.now()

136
server/scanner/LibraryItemScanData.js
View file

@ -2,14 +2,17 @@ const packageJson = require('../../package.json')
const { LogLevel } = require('../utils/constants')
const LibraryItem = require('../models/LibraryItem')
const globals = require('../utils/globals')
const LibraryFile = require('../objects/files/LibraryFile')
const LibraryScan = require('./LibraryScan')
const ScanLogger = require('./ScanLogger')
class LibraryItemScanData {
/**
* @typedef LibraryFileModifiedObject
* @typedef {Object} LibraryFileModifiedObject
* @property {LibraryItem.LibraryFileObject} old
* @property {LibraryItem.LibraryFileObject} new
* @param {{ libraryFolderId: any; libraryId: any; mediaType: any; ino: any; deviceId: any; mtimeMs: any; ctimeMs: any; birthtimeMs: any; path: any; relPath: any; isFile: any; mediaMetadata: any; libraryFiles: any; }} data
*/
constructor(data) {
/** @type {string} */
this.libraryFolderId = data.libraryFolderId
@ -19,6 +22,8 @@ class LibraryItemScanData {
this.mediaType = data.mediaType
/** @type {string} */
this.ino = data.ino
/** @type {string} */
this.deviceId = data.deviceId
/** @type {number} */
this.mtimeMs = data.mtimeMs
/** @type {number} */
@ -54,9 +59,10 @@ class LibraryItemScanData {
*/
get libraryItemObject() {
let size = 0
this.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0))
this.libraryFiles.forEach((lf) => (size += !isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0))
return {
ino: this.ino,
deviceId: this.deviceId,
path: this.path,
relPath: this.relPath,
mediaType: this.mediaType,
@ -80,107 +86,107 @@ class LibraryItemScanData {
/** @type {boolean} */
get hasAudioFileChanges() {
return (this.audioLibraryFilesRemoved.length + this.audioLibraryFilesAdded.length + this.audioLibraryFilesModified.length) > 0
return this.audioLibraryFilesRemoved.length + this.audioLibraryFilesAdded.length + this.audioLibraryFilesModified.length > 0
}
/** @type {LibraryFileModifiedObject[]} */
get audioLibraryFilesModified() {
return this.libraryFilesModified.filter(lf => globals.SupportedAudioTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFilesModified.filter((lf) => globals.SupportedAudioTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryItem.LibraryFileObject[]} */
get audioLibraryFilesRemoved() {
return this.libraryFilesRemoved.filter(lf => globals.SupportedAudioTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFilesRemoved.filter((lf) => globals.SupportedAudioTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryItem.LibraryFileObject[]} */
get audioLibraryFilesAdded() {
return this.libraryFilesAdded.filter(lf => globals.SupportedAudioTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFilesAdded.filter((lf) => globals.SupportedAudioTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryItem.LibraryFileObject[]} */
get audioLibraryFiles() {
return this.libraryFiles.filter(lf => globals.SupportedAudioTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFiles.filter((lf) => globals.SupportedAudioTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryFileModifiedObject[]} */
get imageLibraryFilesModified() {
return this.libraryFilesModified.filter(lf => globals.SupportedImageTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFilesModified.filter((lf) => globals.SupportedImageTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryItem.LibraryFileObject[]} */
get imageLibraryFilesRemoved() {
return this.libraryFilesRemoved.filter(lf => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFilesRemoved.filter((lf) => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryItem.LibraryFileObject[]} */
get imageLibraryFilesAdded() {
return this.libraryFilesAdded.filter(lf => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFilesAdded.filter((lf) => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryItem.LibraryFileObject[]} */
get imageLibraryFiles() {
return this.libraryFiles.filter(lf => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFiles.filter((lf) => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryFileModifiedObject[]} */
get ebookLibraryFilesModified() {
return this.libraryFilesModified.filter(lf => globals.SupportedEbookTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFilesModified.filter((lf) => globals.SupportedEbookTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryItem.LibraryFileObject[]} */
get ebookLibraryFilesRemoved() {
return this.libraryFilesRemoved.filter(lf => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFilesRemoved.filter((lf) => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryItem.LibraryFileObject[]} */
get ebookLibraryFilesAdded() {
return this.libraryFilesAdded.filter(lf => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFilesAdded.filter((lf) => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryItem.LibraryFileObject[]} */
get ebookLibraryFiles() {
return this.libraryFiles.filter(lf => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
return this.libraryFiles.filter((lf) => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
}
/** @type {LibraryItem.LibraryFileObject} */
get descTxtLibraryFile() {
return this.libraryFiles.find(lf => lf.metadata.filename === 'desc.txt')
return this.libraryFiles.find((lf) => lf.metadata.filename === 'desc.txt')
}
/** @type {LibraryItem.LibraryFileObject} */
get readerTxtLibraryFile() {
return this.libraryFiles.find(lf => lf.metadata.filename === 'reader.txt')
return this.libraryFiles.find((lf) => lf.metadata.filename === 'reader.txt')
}
/** @type {LibraryItem.LibraryFileObject} */
get metadataAbsLibraryFile() {
return this.libraryFiles.find(lf => lf.metadata.filename === 'metadata.abs')
return this.libraryFiles.find((lf) => lf.metadata.filename === 'metadata.abs')
}
/** @type {LibraryItem.LibraryFileObject} */
get metadataJsonLibraryFile() {
return this.libraryFiles.find(lf => lf.metadata.filename === 'metadata.json')
return this.libraryFiles.find((lf) => lf.metadata.filename === 'metadata.json')
}
/** @type {LibraryItem.LibraryFileObject} */
get metadataOpfLibraryFile() {
return this.libraryFiles.find(lf => lf.metadata.ext.toLowerCase() === '.opf')
return this.libraryFiles.find((lf) => lf.metadata.ext.toLowerCase() === '.opf')
}
/** @type {LibraryItem.LibraryFileObject} */
get metadataNfoLibraryFile() {
return this.libraryFiles.find(lf => lf.metadata.ext.toLowerCase() === '.nfo')
return this.libraryFiles.find((lf) => lf.metadata.ext.toLowerCase() === '.nfo')
}
/**
*
* @param {LibraryItem} existingLibraryItem
* @param {import('./LibraryScan')} libraryScan
* @returns {boolean} true if changes found
*
* @param {LibraryItem} existingLibraryItem
* @param {import('./LibraryScan') | import('./ScanLogger')} libraryScan
* @returns {Promise<boolean>} true if changes found
*/
async checkLibraryItemData(existingLibraryItem, libraryScan) {
const keysToCompare = ['libraryFolderId', 'ino', 'path', 'relPath', 'isFile']
const keysToCompare = ['libraryFolderId', 'ino', 'deviceId', 'path', 'relPath', 'isFile']
this.hasChanges = false
this.hasPathChange = false
for (const key of keysToCompare) {
@ -219,28 +225,23 @@ class LibraryItemScanData {
this.libraryFilesRemoved = []
this.libraryFilesModified = []
let libraryFilesAdded = this.libraryFiles.map(lf => lf)
let libraryFilesAdded = this.libraryFiles.map((lf) => lf)
for (const existingLibraryFile of existingLibraryItem.libraryFiles) {
// Find matching library file using path first and fallback to using inode value
let matchingLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === existingLibraryFile.metadata.path)
if (!matchingLibraryFile) {
matchingLibraryFile = this.libraryFiles.find(lf => lf.ino === existingLibraryFile.ino)
if (matchingLibraryFile) {
libraryScan.addLog(LogLevel.INFO, `Library file with path "${existingLibraryFile.metadata.path}" not found, but found file with matching inode value "${existingLibraryFile.ino}" at path "${matchingLibraryFile.metadata.path}"`)
}
}
let matchingLibraryFile = this.findMatchingLibraryFileByPathOrInodeAndDeviceId(existingLibraryFile, libraryScan)
if (!matchingLibraryFile) { // Library file removed
if (!matchingLibraryFile) {
// Library file removed
libraryScan.addLog(LogLevel.INFO, `Library file "${existingLibraryFile.metadata.path}" was removed from library item "${existingLibraryItem.relPath}"`)
this.libraryFilesRemoved.push(existingLibraryFile)
existingLibraryItem.libraryFiles = existingLibraryItem.libraryFiles.filter(lf => lf !== existingLibraryFile)
existingLibraryItem.libraryFiles = existingLibraryItem.libraryFiles.filter((lf) => lf !== existingLibraryFile)
this.hasChanges = true
} else {
libraryFilesAdded = libraryFilesAdded.filter(lf => lf !== matchingLibraryFile)
libraryFilesAdded = libraryFilesAdded.filter((lf) => lf !== matchingLibraryFile)
let existingLibraryFileBefore = structuredClone(existingLibraryFile)
if (this.compareUpdateLibraryFile(existingLibraryItem.path, existingLibraryFile, matchingLibraryFile, libraryScan)) {
this.libraryFilesModified.push({old: existingLibraryFileBefore, new: existingLibraryFile})
if (LibraryItemScanData.compareUpdateLibraryFile(existingLibraryItem.path, existingLibraryFile, matchingLibraryFile, libraryScan)) {
this.libraryFilesModified.push({ old: existingLibraryFileBefore, new: existingLibraryFile })
this.hasChanges = true
}
}
@ -263,7 +264,7 @@ class LibraryItemScanData {
if (this.hasChanges) {
existingLibraryItem.size = 0
existingLibraryItem.libraryFiles.forEach((lf) => existingLibraryItem.size += lf.metadata.size)
existingLibraryItem.libraryFiles.forEach((lf) => (existingLibraryItem.size += lf.metadata.size))
existingLibraryItem.lastScan = Date.now()
existingLibraryItem.lastScanVersion = packageJson.version
@ -274,25 +275,25 @@ class LibraryItemScanData {
existingLibraryItem.changed('libraryFiles', true)
}
await existingLibraryItem.save()
return true
}
return false
return this.hasChanges
}
/**
* Update existing library file with scanned in library file data
* @param {string} libraryItemPath
* @param {LibraryItem.LibraryFileObject} existingLibraryFile
* @param {import('../objects/files/LibraryFile')} scannedLibraryFile
* @param {import('./LibraryScan')} libraryScan
* @param {LibraryItem.LibraryFileObject} existingLibraryFile
* @param {import('../objects/files/LibraryFile')} scannedLibraryFile
* @param {import('./LibraryScan') | import('./ScanLogger')} libraryScan
* @returns {boolean} false if no changes
*/
compareUpdateLibraryFile(libraryItemPath, existingLibraryFile, scannedLibraryFile, libraryScan) {
static compareUpdateLibraryFile(libraryItemPath, existingLibraryFile, scannedLibraryFile, libraryScan) {
let hasChanges = false
if (existingLibraryFile.ino !== scannedLibraryFile.ino) {
if (existingLibraryFile.ino !== scannedLibraryFile.ino && existingLibraryFile.deviceId !== scannedLibraryFile.deviceId) {
existingLibraryFile.ino = scannedLibraryFile.ino
existingLibraryFile.deviceId = scannedLibraryFile.deviceId
hasChanges = true
}
@ -315,40 +316,57 @@ class LibraryItemScanData {
return hasChanges
}
/**
* @returns {LibraryFile | undefined} if [existingLibraryFile] matches an existing libraryFile
* @param {LibraryItem.LibraryFileObject} [existingLibraryFile]
* @param {LibraryScan | ScanLogger} [libraryScan]
*/
findMatchingLibraryFileByPathOrInodeAndDeviceId(existingLibraryFile, libraryScan) {
if (!existingLibraryFile) return
let matchingLibraryFile = this.libraryFiles.find((lf) => lf.metadata.path === existingLibraryFile.metadata.path)
if (!matchingLibraryFile) {
matchingLibraryFile = this.libraryFiles.find((lf) => lf.ino === existingLibraryFile.ino && lf.deviceId === existingLibraryFile.deviceId)
if (matchingLibraryFile) {
libraryScan && libraryScan.addLog(LogLevel.INFO, `Library file with path "${existingLibraryFile.metadata.path}" not found, but found file with matching inode value "${existingLibraryFile.ino}" at path "${matchingLibraryFile.metadata.path}"`)
}
}
return matchingLibraryFile
}
/**
* Check if existing audio file on Book was removed
* @param {import('../models/Book').AudioFileObject} existingAudioFile
* @param {import('../models/Book').AudioFileObject} existingAudioFile
* @returns {boolean} true if audio file was removed
*/
checkAudioFileRemoved(existingAudioFile) {
if (!this.audioLibraryFilesRemoved.length) return false
// First check exact path
if (this.audioLibraryFilesRemoved.some(af => af.metadata.path === existingAudioFile.metadata.path)) {
if (this.audioLibraryFilesRemoved.some((af) => af.metadata.path === existingAudioFile.metadata.path)) {
return true
}
// Fallback to check inode value
return this.audioLibraryFilesRemoved.some(af => af.ino === existingAudioFile.ino)
return this.audioLibraryFilesRemoved.some((af) => af.ino === existingAudioFile.ino && af.deviceId === existingAudioFile.deviceId)
}
/**
* Check if existing ebook file on Book was removed
* @param {import('../models/Book').EBookFileObject} ebookFile
* @param {import('../models/Book').EBookFileObject} ebookFile
* @returns {boolean} true if ebook file was removed
*/
checkEbookFileRemoved(ebookFile) {
if (!this.ebookLibraryFiles.length) return true
if (!this.ebookLibraryFilesRemoved.length) return false
if (this.ebookLibraryFiles.some(lf => lf.metadata.path === ebookFile.metadata.path)) {
return false
if (this.ebookLibraryFilesRemoved.some((lf) => lf.metadata.path === ebookFile.metadata.path)) {
return true
}
return !this.ebookLibraryFiles.some(lf => lf.ino === ebookFile.ino)
return this.ebookLibraryFilesRemoved.some((lf) => lf.ino === ebookFile.ino && lf.deviceId === ebookFile.deviceId)
}
/**
* Set data parsed from filenames
*
* @param {Object} bookMetadata
*
* @param {Object} bookMetadata
*/
setBookMetadataFromFilenames(bookMetadata) {
const keysToMap = ['title', 'subtitle', 'publishedYear', 'asin']
@ -374,4 +392,4 @@ class LibraryItemScanData {
}
}
}
module.exports = LibraryItemScanData
module.exports = LibraryItemScanData

47
server/scanner/LibraryItemScanner.js
View file

@ -23,7 +23,7 @@ class LibraryItemScanner {
* Scan single library item
*
* @param {string} libraryItemId
* @param {{relPath:string, path:string}} [updateLibraryItemDetails] used by watcher when item folder was renamed
* @param {{relPath:string, path:string, isFile: boolean}} [updateLibraryItemDetails] used by watcher when item folder was renamed
* @returns {number} ScanResult
*/
async scanLibraryItem(libraryItemId, updateLibraryItemDetails = null) {
@ -139,24 +139,11 @@ class LibraryItemScanner {
const newLibraryFile = new LibraryFile()
// fileItem.path is the relative path
await newLibraryFile.setDataFromPath(fileItem.fullpath, fileItem.path)
// TODO: BUGBUG - this is pushing the object, not a JSON string of the object like elsewhere
libraryFiles.push(newLibraryFile)
}
const libraryItemStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)
return new LibraryItemScanData({
libraryFolderId: folder.id,
libraryId: library.id,
mediaType: library.mediaType,
ino: libraryItemStats.ino,
mtimeMs: libraryItemStats.mtimeMs || 0,
ctimeMs: libraryItemStats.ctimeMs || 0,
birthtimeMs: libraryItemStats.birthtimeMs || 0,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
isFile: isSingleMediaItem,
mediaMetadata: libraryItemData.mediaMetadata || null,
libraryFiles
})
return await buildLibraryItemScanData(libraryItemData, folder, library, isSingleMediaItem, libraryFiles)
}
/**
@ -201,7 +188,7 @@ class LibraryItemScanner {
* @param {import('../models/Library')} library
* @param {import('../models/LibraryFolder')} folder
* @param {boolean} isSingleMediaItem
* @returns {Promise<LibraryItem>} ScanResult
* @returns {Promise<LibraryItem | null>} ScanResult
*/
async scanPotentialNewLibraryItem(libraryItemPath, library, folder, isSingleMediaItem) {
const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, isSingleMediaItem)
@ -219,3 +206,29 @@ class LibraryItemScanner {
}
}
module.exports = new LibraryItemScanner()
/**
* @param {{ path?: any; relPath?: any; mediaMetadata?: any; }} libraryItemData
* @param {import("../models/LibraryFolder")} folder
* @param {import("../models/Library")} library
* @param {boolean} isSingleMediaItem
* @param {LibraryFile[]} libraryFiles
*/
async function buildLibraryItemScanData(libraryItemData, folder, library, isSingleMediaItem, libraryFiles) {
const libraryItemStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)
return new LibraryItemScanData({
libraryFolderId: folder.id,
libraryId: library.id,
mediaType: library.mediaType,
ino: libraryItemStats.ino,
deviceId: libraryItemStats.dev,
mtimeMs: libraryItemStats.mtimeMs || 0,
ctimeMs: libraryItemStats.ctimeMs || 0,
birthtimeMs: libraryItemStats.birthtimeMs || 0,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
isFile: isSingleMediaItem,
mediaMetadata: libraryItemData.mediaMetadata || null,
libraryFiles
})
}

113
server/scanner/LibraryScanner.js
View file

@ -297,7 +297,7 @@ class LibraryScanner {
* Get scan data for library folder
* @param {import('../models/Library')} library
* @param {import('../models/LibraryFolder')} folder
* @returns {LibraryItemScanData[]}
* @returns {Promise<LibraryItemScanData[]>}
*/
async scanFolder(library, folder) {
const folderPath = fileUtils.filePathToPOSIX(folder.path)
@ -344,22 +344,7 @@ class LibraryScanner {
continue
}
items.push(
new LibraryItemScanData({
libraryFolderId: folder.id,
libraryId: folder.libraryId,
mediaType: library.mediaType,
ino: libraryItemFolderStats.ino,
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
isFile,
mediaMetadata: libraryItemData.mediaMetadata || null,
libraryFiles: fileObjs
})
)
items.push(createLibraryItemScanData(folder, library, libraryItemFolderStats, libraryItemData, isFile, fileObjs))
}
return items
}
@ -642,12 +627,25 @@ class LibraryScanner {
}
module.exports = new LibraryScanner()
/**
* @param {import("../models/LibraryItem") | LibraryItemScanData} libraryItem1
* @param {import("../models/LibraryItem") | LibraryItemScanData} libraryItem2
*/
function ItemToFileInoMatch(libraryItem1, libraryItem2) {
return libraryItem1.isFile && libraryItem2.libraryFiles.some((lf) => lf.ino === libraryItem1.ino)
return (
libraryItem1.isFile &&
libraryItem2.libraryFiles.some((lf) => {
return lf.ino === libraryItem1.ino && lf.deviceId === libraryItem1.deviceId
})
)
}
/**
* @param {LibraryItemScanData} libraryItem1
* @param {import("../models/LibraryItem")} libraryItem2
*/
function ItemToItemInoMatch(libraryItem1, libraryItem2) {
return libraryItem1.ino === libraryItem2.ino
return libraryItem1.ino === libraryItem2.ino && libraryItem1.deviceId === libraryItem2.deviceId
}
function hasAudioFiles(fileUpdateGroup, itemDir) {
@ -658,54 +656,111 @@ function isSingleMediaFile(fileUpdateGroup, itemDir) {
return itemDir === fileUpdateGroup[itemDir]
}
/**
* @param {UUIDV4} libraryId
* @param {string} fullPath
* @returns {Promise<import('../models/LibraryItem').LibraryItemExpanded | null>} library item that matches
*/
async function findLibraryItemByItemToItemInoMatch(libraryId, fullPath) {
const ino = await fileUtils.getIno(fullPath)
const deviceId = await fileUtils.getDeviceId(fullPath)
if (!ino) return null
const existingLibraryItem = await Database.libraryItemModel.findOneExpanded({
libraryId: libraryId,
ino: ino
ino: ino,
deviceId: deviceId
})
if (existingLibraryItem) Logger.debug(`[LibraryScanner] Found library item with matching inode "${ino}" at path "${existingLibraryItem.path}"`)
return existingLibraryItem
}
/**
* @param {UUIDV4} libraryId
* @param {string} fullPath
* @param {boolean} isSingleMedia
* @returns {Promise<import('../models/LibraryItem').LibraryItemExpanded | null>} library item that matches
*/
async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingleMedia) {
if (!isSingleMedia) return null
// check if it was moved from another folder by comparing the ino to the library files
const ino = await fileUtils.getIno(fullPath)
const deviceId = await fileUtils.getDeviceId(fullPath)
if (!ino) return null
const existingLibraryItem = await Database.libraryItemModel.findOneExpanded(
[
{
libraryId: libraryId
},
sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), {
sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode AND json_each.value->>"$.deviceId" = :deviceId)'), {
[sequelize.Op.gt]: 0
})
],
{
inode: ino
inode: ino,
deviceId: deviceId
}
)
if (existingLibraryItem) Logger.debug(`[LibraryScanner] Found library item with a library file matching inode "${ino}" at path "${existingLibraryItem.path}"`)
return existingLibraryItem
}
/**
* @param {UUIDV4} libraryId
* @param {string} fullPath
* @param {boolean} isSingleMedia
* @param {string[]} itemFiles
* @returns {Promise<import('../models/LibraryItem').LibraryItemExpanded | null>} library item that matches
*/
async function findLibraryItemByFileToItemInoMatch(libraryId, fullPath, isSingleMedia, itemFiles) {
if (isSingleMedia) return null
// check if it was moved from the root folder by comparing the ino to the ino of the scanned files
// check if it was moved from the root folder by comparing the ino and deviceId to the ino and deviceId of the scanned files
let itemFileInos = []
for (const itemFile of itemFiles) {
const ino = await fileUtils.getIno(Path.posix.join(fullPath, itemFile))
if (ino) itemFileInos.push(ino)
const deviceId = await fileUtils.getDeviceId(Path.posix.join(fullPath, itemFile))
if (ino && deviceId) itemFileInos.push({ ino: ino, deviceId: deviceId })
}
if (!itemFileInos.length) return null
const existingLibraryItem = await Database.libraryItemModel.findOneExpanded({
libraryId: libraryId,
ino: {
[sequelize.Op.in]: itemFileInos
/** @type {import('../models/LibraryItem').LibraryItemExpanded | null} */
let existingLibraryItem = null
for (let item in itemFileInos) {
existingLibraryItem = await Database.libraryItemModel.findOneExpanded({
libraryId: libraryId,
[sequelize.Op.or]: itemFileInos
})
if (existingLibraryItem) {
break
}
})
}
if (existingLibraryItem) Logger.debug(`[LibraryScanner] Found library item with inode matching one of "${itemFileInos.join(',')}" at path "${existingLibraryItem.path}"`)
return existingLibraryItem
}
/**
* @param {{ id: any; libraryId: any; }} folder
* @param {{ mediaType: any; }} library
* @param {{ ino: any; dev: any; mtimeMs: any; ctimeMs: any; birthtimeMs: any; }} libraryItemFolderStats
* @param {{ path: any; relPath: any; mediaMetadata: any; }} libraryItemData
* @param {any} isFile
* @param {any} fileObjs
* @returns {LibraryItemScanData} new object
*/
function createLibraryItemScanData(folder, library, libraryItemFolderStats, libraryItemData, isFile, fileObjs) {
return new LibraryItemScanData({
libraryFolderId: folder.id,
libraryId: folder.libraryId,
mediaType: library.mediaType,
ino: libraryItemFolderStats.ino,
deviceId: libraryItemFolderStats.dev,
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
isFile,
mediaMetadata: libraryItemData.mediaMetadata || null,
libraryFiles: fileObjs
})
}

2
server/scanner/PodcastScanner.js
View file

@ -367,7 +367,7 @@ class PodcastScanner {
* @param {PodcastEpisode[]} podcastEpisodes Not the models for new podcasts
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {import('./LibraryScan')} libraryScan
* @param {string} [existingLibraryItemId]
* @param {string | null} [existingLibraryItemId]
* @returns {Promise<PodcastMetadataObject>}
*/
async getPodcastMetadataFromScanData(podcastEpisodes, libraryItemData, libraryScan, existingLibraryItemId = null) {

42
server/utils/fileUtils.js
View file

@ -47,6 +47,10 @@ function getFileStat(path) {
}
}
/**
* @param {string} path
* @returns {Promise<object | null>}
*/
async function getFileTimestampsWithIno(path) {
try {
var stat = await fs.stat(path, { bigint: true })
@ -55,11 +59,12 @@ async function getFileTimestampsWithIno(path) {
mtimeMs: Number(stat.mtimeMs),
ctimeMs: Number(stat.ctimeMs),
birthtimeMs: Number(stat.birthtimeMs),
ino: String(stat.ino)
ino: String(stat.ino),
deviceId: String(stat.dev)
}
} catch (err) {
Logger.error(`[fileUtils] Failed to getFileTimestampsWithIno for path "${path}"`, err)
return false
return null
}
}
module.exports.getFileTimestampsWithIno = getFileTimestampsWithIno
@ -92,7 +97,7 @@ module.exports.getFileMTimeMs = async (path) => {
/**
*
* @param {string} filepath
* @returns {boolean}
* @returns {Promise<boolean>} isFile
*/
async function checkPathIsFile(filepath) {
try {
@ -104,6 +109,10 @@ async function checkPathIsFile(filepath) {
}
module.exports.checkPathIsFile = checkPathIsFile
/**
* @param {string} path
* @returns {string | null} inode
*/
function getIno(path) {
return fs
.stat(path, { bigint: true })
@ -115,10 +124,25 @@ function getIno(path) {
}
module.exports.getIno = getIno
/**
* @param {string} path
* @returns {Promise<string | null>} deviceId
*/
async function getDeviceId(path) {
try {
var data = await fs.stat(path)
return String(data.dev)
} catch (error) {
Logger.error(`[Utils] Failed to get device Id for path "${path}": ${error}`)
return null
}
}
module.exports.getDeviceId = getDeviceId
/**
* Read contents of file
* @param {string} path
* @returns {string}
* @returns {Promise<string>} file contents
*/
async function readTextFile(path) {
try {
@ -135,7 +159,7 @@ module.exports.readTextFile = readTextFile
* Check if file or directory should be ignored. Returns a string of the reason to ignore, or null if not ignored
*
* @param {string} path
* @returns {string}
* @returns {string | null} reason to ignore
*/
module.exports.shouldIgnoreFile = (path) => {
// Check if directory or file name starts with "."
@ -178,8 +202,8 @@ module.exports.shouldIgnoreFile = (path) => {
/**
* Get array of files inside dir
* @param {string} path
* @param {string} [relPathToReplace]
* @returns {FilePathItem[]}
* @param {string | null} [relPathToReplace]
* @returns {Promise<FilePathItem[]>}
*/
module.exports.recurseFiles = async (path, relPathToReplace = null) => {
path = filePathToPOSIX(path)
@ -219,6 +243,8 @@ module.exports.recurseFiles = async (path, relPathToReplace = null) => {
item.fullname = filePathToPOSIX(item.fullname)
item.path = filePathToPOSIX(item.path)
// BUGBUG: This is broken with symlinked directory /tmp -> /private/tmp. when library is in /tmp/testLibrary, it tries to replace /tmp/testLibrary with '' but in a canonical path (non-symlinked)
// TODO: find the commit that added relPathToReplace and figure out what it's trying to do and make it do that properly
const relpath = item.fullname.replace(relPathToReplace, '')
let reldirname = Path.dirname(relpath)
if (reldirname === '.') reldirname = ''
@ -292,7 +318,7 @@ module.exports.getFilePathItemFromFileUpdate = (fileUpdate) => {
*
* @param {string} url
* @param {string} filepath path to download the file to
* @param {Function} [contentTypeFilter] validate content type before writing
* @param {Function | null} [contentTypeFilter] validate content type before writing
* @returns {Promise}
*/
module.exports.downloadFile = (url, filepath, contentTypeFilter = null) => {

45
server/utils/migrations/absMetadataMigration.js
View file

@ -5,17 +5,17 @@ const fileUtils = require('../fileUtils')
const LibraryFile = require('../../objects/files/LibraryFile')
/**
*
* @param {import('../../models/LibraryItem')} libraryItem
*
* @param {import('../../models/LibraryItem')} libraryItem
* @returns {Promise<boolean>} false if failed
*/
async function writeMetadataFileForItem(libraryItem) {
const storeMetadataWithItem = global.ServerSettings.storeMetadataWithItem && !libraryItem.isFile
const metadataPath = storeMetadataWithItem ? libraryItem.path : Path.join(global.MetadataPath, 'items', libraryItem.id)
const metadataFilepath = fileUtils.filePathToPOSIX(Path.join(metadataPath, 'metadata.json'))
if ((await fsExtra.pathExists(metadataFilepath))) {
if (await fsExtra.pathExists(metadataFilepath)) {
// Metadata file already exists do nothing
return null
return false
}
Logger.info(`[absMetadataMigration] metadata file not found at "${metadataFilepath}" - creating`)
@ -27,20 +27,24 @@ async function writeMetadataFileForItem(libraryItem) {
const metadataJson = libraryItem.media.getAbsMetadataJson()
// Save to file
const success = await fsExtra.writeFile(metadataFilepath, JSON.stringify(metadataJson, null, 2)).then(() => true).catch((error) => {
Logger.error(`[absMetadataMigration] failed to save metadata file at "${metadataFilepath}"`, error.message || error)
return false
})
const success = await fsExtra
.writeFile(metadataFilepath, JSON.stringify(metadataJson, null, 2))
.then(() => true)
.catch((error) => {
Logger.error(`[absMetadataMigration] failed to save metadata file at "${metadataFilepath}"`, error.message || error)
return false
})
if (!success) return false
if (!storeMetadataWithItem) return true // No need to do anything else
// Safety check to make sure library file with the same path isnt already there
libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== metadataFilepath)
libraryItem.libraryFiles = libraryItem.libraryFiles.filter((lf) => lf.metadata.path !== metadataFilepath)
// Put new library file in library item
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilepath, 'metadata.json')
// TODO: BUGBUG - this shouldn't be JSON and it may not be the right type LibraryFileObject
libraryItem.libraryFiles.push(newLibraryFile.toJSON())
// Update library item timestamps and total size
@ -49,20 +53,23 @@ async function writeMetadataFileForItem(libraryItem) {
libraryItem.mtime = libraryItemDirTimestamps.mtimeMs
libraryItem.ctime = libraryItemDirTimestamps.ctimeMs
let size = 0
libraryItem.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0))
libraryItem.libraryFiles.forEach((lf) => (size += !isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0))
libraryItem.size = size
}
libraryItem.changed('libraryFiles', true)
return libraryItem.save().then(() => true).catch((error) => {
Logger.error(`[absMetadataMigration] failed to save libraryItem "${libraryItem.id}"`, error.message || error)
return false
})
return libraryItem
.save()
.then(() => true)
.catch((error) => {
Logger.error(`[absMetadataMigration] failed to save libraryItem "${libraryItem.id}"`, error.message || error)
return false
})
}
/**
*
* @param {import('../../Database')} Database
*
* @param {import('../../Database')} Database
* @param {number} [offset=0]
* @param {number} [totalCreated=0]
*/
@ -83,11 +90,11 @@ async function runMigration(Database, offset = 0, totalCreated = 0) {
}
/**
*
* @param {import('../../Database')} Database
*
* @param {import('../../Database')} Database
*/
module.exports.migrate = async (Database) => {
Logger.info(`[absMetadataMigration] Starting metadata.json migration`)
const totalCreated = await runMigration(Database)
Logger.info(`[absMetadataMigration] Finished metadata.json migration (${totalCreated} files created)`)
}
}