diff --git a/Dockerfile b/Dockerfile index 816bdd3c3..3ddc1a9d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,6 +50,7 @@ ARG NUSQLITE3_PATH RUN apk add --no-cache --update \ tzdata \ ffmpeg \ + postgresql-client \ tini WORKDIR /app diff --git a/package-lock.json b/package-lock.json index 47d882171..bc23a15c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,8 @@ "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", + "pg": "^8.19.0", + "pg-hstore": "^2.3.4", "semver": "^7.6.3", "sequelize": "^6.35.2", "socket.io": "^4.5.4", @@ -4138,10 +4140,106 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" }, + "node_modules/pg": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", + "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.15.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, "node_modules/pg-connection-string": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", - "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-hstore": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz", + "integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==", + "license": "MIT", + "dependencies": { + "underscore": "^1.13.1" + }, + "engines": { + "node": ">= 0.8.x" + } + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz", + "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } }, "node_modules/picocolors": { "version": "1.0.0", @@ -4173,6 +4271,45 @@ "node": ">=8" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -4946,6 +5083,15 @@ "node": ">=8" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -5265,6 +5411,12 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "node_modules/underscore": { + "version": "1.13.8", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", + "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", + "license": "MIT" + }, "node_modules/unique-filename": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", @@ -5476,6 +5628,15 @@ "node": ">=4.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 534444bb1..b2c76081e 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "docker": "docker buildx build --platform linux/amd64,linux/arm64 --push . -t advplyr/audiobookshelf", "docker-amd64-local": "docker buildx build --platform linux/amd64 --load . -t advplyr/audiobookshelf-amd64-local", "docker-arm64-local": "docker buildx build --platform linux/arm64 --load . -t advplyr/audiobookshelf-arm64-local", + "migrate-sqlite-to-postgres": "node server/scripts/migrateSqliteToPostgres.js", "deploy-linux": "node deploy/linux", "test": "mocha", "coverage": "nyc mocha" @@ -52,6 +53,8 @@ "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", + "pg": "^8.19.0", + "pg-hstore": "^2.3.4", "semver": "^7.6.3", "sequelize": "^6.35.2", "socket.io": "^4.5.4", diff --git a/server/Database.js b/server/Database.js index 213c2c61b..df0d6939b 100644 --- a/server/Database.js +++ b/server/Database.js @@ -14,6 +14,7 @@ class Database { constructor() { this.sequelize = null this.dbPath = null + this.dialect = 'sqlite' this.isNew = false // New absdatabase.sqlite created this.hasRootUser = false // Used to show initialization page in web ui @@ -162,11 +163,37 @@ class Database { return this.models.device } + /** + * @returns {'sqlite'|'postgres'} + */ + getConfiguredDialect() { + const explicitDialect = process.env.DB_DIALECT?.trim()?.toLowerCase() + if (explicitDialect === 'postgres' || explicitDialect === 'sqlite') { + return explicitDialect + } + + const databaseUrl = process.env.DATABASE_URL || '' + if (databaseUrl.startsWith('postgres://') || databaseUrl.startsWith('postgresql://')) { + return 'postgres' + } + + return 'sqlite' + } + + isSqliteDialect() { + return this.dialect === 'sqlite' + } + + isPostgresDialect() { + return this.dialect === 'postgres' + } + /** * Check if db file exists * @returns {boolean} */ async checkHasDb() { + if (!this.isSqliteDialect()) return true if (!(await fs.pathExists(this.dbPath))) { Logger.info(`[Database] absdatabase.sqlite not found at ${this.dbPath}`) return false @@ -174,20 +201,39 @@ class Database { return true } + /** + * Check if any user tables exist (for networked dialects) + * @returns {Promise} + */ + async checkHasTables() { + const queryInterface = this.sequelize.getQueryInterface() + const tables = await queryInterface.showAllTables() + return Array.isArray(tables) && tables.length > 0 + } + /** * Connect to db, build models and run migrations * @param {boolean} [force=false] Used for testing, drops & re-creates all tables */ async init(force = false) { - this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') - - // First check if this is a new database - this.isNew = !(await this.checkHasDb()) || force + this.dialect = this.getConfiguredDialect() + if (this.isSqliteDialect()) { + this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') + // First check if this is a new database + this.isNew = !(await this.checkHasDb()) || force + } else { + this.dbPath = process.env.DATABASE_URL || null + this.isNew = !!force + } if (!(await this.connect())) { throw new Error('Database connection failed') } + if (this.isPostgresDialect() && !force) { + this.isNew = !(await this.checkHasTables()) + } + try { const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath) await migrationManager.init(packageJson.version) @@ -214,7 +260,11 @@ class Database { * @returns {boolean} */ async connect() { - Logger.info(`[Database] Initializing db at "${this.dbPath}"`) + if (this.isSqliteDialect()) { + Logger.info(`[Database] Initializing sqlite db at "${this.dbPath}"`) + } else { + Logger.info(`[Database] Initializing postgres db connection`) + } let logging = false let benchmark = false @@ -229,13 +279,27 @@ class Database { benchmark = true } - this.sequelize = new Sequelize({ - dialect: 'sqlite', - storage: this.dbPath, - logging: logging, - benchmark: benchmark, - transactionType: 'IMMEDIATE' - }) + if (this.isSqliteDialect()) { + this.sequelize = new Sequelize({ + dialect: 'sqlite', + storage: this.dbPath, + logging: logging, + benchmark: benchmark, + transactionType: 'IMMEDIATE' + }) + } else { + if (!process.env.DATABASE_URL) { + Logger.error(`[Database] DATABASE_URL is required when DB_DIALECT=postgres`) + return false + } + + this.sequelize = new Sequelize(process.env.DATABASE_URL, { + dialect: 'postgres', + logging: logging, + benchmark: benchmark, + quoteIdentifiers: false + }) + } // Helper function this.sequelize.uppercaseFirst = (str) => (str ? `${str[0].toUpperCase()}${str.substr(1)}` : '') @@ -243,34 +307,36 @@ class Database { try { await this.sequelize.authenticate() - // Set SQLite pragmas from environment variables - const allowedPragmas = [ - { name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' }, - { name: 'cache_size', env: 'SQLITE_CACHE_SIZE' }, - { name: 'temp_store', env: 'SQLITE_TEMP_STORE' } - ] + if (this.isSqliteDialect()) { + // Set SQLite pragmas from environment variables + const allowedPragmas = [ + { name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' }, + { name: 'cache_size', env: 'SQLITE_CACHE_SIZE' }, + { name: 'temp_store', env: 'SQLITE_TEMP_STORE' } + ] - for (const pragma of allowedPragmas) { - const value = process.env[pragma.env] - if (value !== undefined) { - try { - Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`) - await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`) - const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`) - Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result) - } catch (error) { - Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error) + for (const pragma of allowedPragmas) { + const value = process.env[pragma.env] + if (value !== undefined) { + try { + Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`) + await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`) + const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`) + Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result) + } catch (error) { + Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error) + } } } - } - if (process.env.NUSQLITE3_PATH) { - await this.loadExtension(process.env.NUSQLITE3_PATH) - Logger.info(`[Database] Db supports unaccent and unicode foldings`) - this.supportsUnaccent = true - this.supportsUnicodeFoldings = true + if (process.env.NUSQLITE3_PATH) { + await this.loadExtension(process.env.NUSQLITE3_PATH) + Logger.info(`[Database] Db supports unaccent and unicode foldings`) + this.supportsUnaccent = true + this.supportsUnicodeFoldings = true + } } - Logger.info(`[Database] Db connection was successful`) + Logger.info(`[Database] Db connection was successful (${this.dialect})`) return true } catch (error) { Logger.error(`[Database] Failed to connect to db`, error) @@ -307,7 +373,7 @@ class Database { * Disconnect from db */ async disconnect() { - Logger.info(`[Database] Disconnecting sqlite db`) + Logger.info(`[Database] Disconnecting ${this.dialect} db`) await this.sequelize.close() } @@ -315,7 +381,7 @@ class Database { * Reconnect to db and init */ async reconnect() { - Logger.info(`[Database] Reconnecting sqlite db`) + Logger.info(`[Database] Reconnecting ${this.dialect} db`) await this.init() } @@ -346,6 +412,11 @@ class Database { require('./models/CustomMetadataProvider').init(this.sequelize) require('./models/MediaItemShare').init(this.sequelize) + if (this.isPostgresDialect() && !force && !this.isNew) { + Logger.info('[Database] Skipping sequelize.sync for existing postgres schema') + return Promise.resolve() + } + return this.sequelize.sync({ force, alter: false }) } @@ -368,7 +439,7 @@ class Database { * Loads most of the data from the database. This is a temporary solution. */ async loadData() { - if (this.isNew && (await dbMigration.checkShouldMigrate())) { + if (this.isSqliteDialect() && this.isNew && (await dbMigration.checkShouldMigrate())) { Logger.info(`[Database] New database was created and old database was detected - migrating old to new`) await dbMigration.migrate(this.models) } @@ -441,9 +512,19 @@ class Database { return this.models.setting.updateSettingObj(settings.toJSON()) } - getPlaybackSessions(where = null) { + getPlaybackSessions(where = null, options = null) { if (!this.sequelize) return false - return this.models.playbackSession.getOldPlaybackSessions(where) + return this.models.playbackSession.getOldPlaybackSessions(where, options || undefined) + } + + countPlaybackSessions(where = null) { + if (!this.sequelize) return false + return this.models.playbackSession.countWithWhere(where) + } + + getPlaybackSessionsForStats(where = null) { + if (!this.sequelize) return false + return this.models.playbackSession.getPlaybackSessionsForStats(where) } getPlaybackSession(sessionId) { @@ -847,6 +928,14 @@ WHERE EXISTS ( * It adds triggers to update libraryItems.title[IgnorePrefix] when (books|podcasts).title[IgnorePrefix] is updated */ async addTriggers() { + if (this.isPostgresDialect()) { + return this.addPostgresTriggers() + } + if (!this.isSqliteDialect()) { + Logger.info(`[Database] Skipping sqlite-only triggers for dialect ${this.dialect}`) + return + } + await this.addTriggerIfNotExists('books', 'title', 'id', 'libraryItems', 'title', 'mediaId') await this.addTriggerIfNotExists('books', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId') await this.addTriggerIfNotExists('podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId') @@ -944,6 +1033,121 @@ WHERE EXISTS ( await addAuthorsUpdateTriggerIfNotExists() } + /** + * Postgres equivalent of the sqlite libraryItems denormalization triggers above. + * Identifiers are unquoted (and therefore folded to lowercase by postgres) to match + * the quoteIdentifiers: false strategy used for the postgres dialect. + */ + async addPostgresTriggers() { + Logger.info('[Database] Adding postgres denormalization triggers') + + await this.addPostgresTitleTriggerIfNotExists('books', 'title') + await this.addPostgresTitleTriggerIfNotExists('books', 'titleIgnorePrefix') + await this.addPostgresTitleTriggerIfNotExists('podcasts', 'title') + await this.addPostgresTitleTriggerIfNotExists('podcasts', 'titleIgnorePrefix') + await this.addPostgresAuthorNamesTriggersIfNotExist() + } + + async postgresTriggerExists(triggerName) { + const [[{ count }]] = await this.sequelize.query(`SELECT COUNT(*) as count FROM pg_trigger WHERE NOT tgisinternal AND tgname = '${triggerName}'`) + return Number(count) > 0 + } + + async addPostgresTitleTriggerIfNotExists(sourceTable, sourceColumn) { + const foldedColumn = sourceColumn.toLowerCase() + const action = `update_libraryItems_${sourceColumn}` + const fromSource = sourceTable === 'books' ? '' : `_from_${sourceTable}_${sourceColumn}` + const triggerName = this.convertToSnakeCase(`${action}${fromSource}`) + const functionName = `${triggerName}_fn` + + if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists + + Logger.info(`[Database] Adding trigger ${triggerName}`) + + await this.sequelize.query(` + CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$ + BEGIN + UPDATE libraryitems + SET ${foldedColumn} = NEW.${foldedColumn} + WHERE mediaid = NEW.id; + RETURN NEW; + END; + $func$ LANGUAGE plpgsql + `) + await this.sequelize.query(` + CREATE TRIGGER ${triggerName} + AFTER UPDATE OF ${foldedColumn} ON ${sourceTable} + FOR EACH ROW + EXECUTE FUNCTION ${functionName}() + `) + } + + async addPostgresAuthorNamesTriggersIfNotExist() { + // string_agg is the postgres equivalent of sqlite GROUP_CONCAT; both return NULL for an empty set + const authorNamesSubQuery = (bookIdExpression) => ` + SELECT string_agg(authors.name, ', ' ORDER BY bookauthors.createdat ASC), string_agg(authors.lastfirst, ', ' ORDER BY bookauthors.createdat ASC) + FROM authors JOIN bookauthors ON authors.id = bookauthors.authorid + WHERE bookauthors.bookid = ${bookIdExpression} + ` + + const addBookAuthorsTriggerIfNotExists = async (action) => { + const modifiedRecord = action === 'delete' ? 'OLD' : 'NEW' + const triggerName = this.convertToSnakeCase(`update_libraryItems_authorNames_on_bookAuthors_${action}`) + const functionName = `${triggerName}_fn` + + if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists + + Logger.info(`[Database] Adding trigger ${triggerName}`) + + await this.sequelize.query(` + CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$ + BEGIN + UPDATE libraryitems + SET (authornamesfirstlast, authornameslastfirst) = (${authorNamesSubQuery(`${modifiedRecord}.bookid`)}) + WHERE mediaid = ${modifiedRecord}.bookid; + RETURN ${modifiedRecord}; + END; + $func$ LANGUAGE plpgsql + `) + await this.sequelize.query(` + CREATE TRIGGER ${triggerName} + AFTER ${action.toUpperCase()} ON bookauthors + FOR EACH ROW + EXECUTE FUNCTION ${functionName}() + `) + } + + const addAuthorsUpdateTriggerIfNotExists = async () => { + const triggerName = this.convertToSnakeCase('update_libraryItems_authorNames_on_authors_update') + const functionName = `${triggerName}_fn` + + if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists + + Logger.info(`[Database] Adding trigger ${triggerName}`) + + await this.sequelize.query(` + CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$ + BEGIN + UPDATE libraryitems + SET (authornamesfirstlast, authornameslastfirst) = (${authorNamesSubQuery('libraryitems.mediaid')}) + WHERE mediaid IN (SELECT bookid FROM bookauthors WHERE authorid = NEW.id); + RETURN NEW; + END; + $func$ LANGUAGE plpgsql + `) + await this.sequelize.query(` + CREATE TRIGGER ${triggerName} + AFTER UPDATE OF name ON authors + FOR EACH ROW + EXECUTE FUNCTION ${functionName}() + `) + } + + await addBookAuthorsTriggerIfNotExists('insert') + await addBookAuthorsTriggerIfNotExists('delete') + await addAuthorsUpdateTriggerIfNotExists() + } + convertToSnakeCase(str) { return str.replace(/([A-Z])/g, '_$1').toLowerCase() } @@ -954,6 +1158,7 @@ WHERE EXISTS ( this.supportsUnaccent = supportsUnaccent this.query = query this.hasAccents = false + this.dialect = sequelize.getDialect() } /** @@ -989,9 +1194,10 @@ WHERE EXISTS ( */ matchExpression(column) { const pattern = this.sequelize.escape(`%${this.query}%`) - if (!this.supportsUnaccent) return `${column} LIKE ${pattern}` + const likeOperator = this.dialect === 'postgres' ? 'ILIKE' : 'LIKE' + if (!this.supportsUnaccent) return `${column} ${likeOperator} ${pattern}` const normalizedColumn = this.hasAccents ? column : this.normalize(column) - return `${normalizedColumn} LIKE ${pattern}` + return `${normalizedColumn} ${likeOperator} ${pattern}` } } } diff --git a/server/Server.js b/server/Server.js index c1657ff1e..b3300344a 100644 --- a/server/Server.js +++ b/server/Server.js @@ -479,14 +479,27 @@ class Server { // Remove series from hide from continue listening that no longer exist try { - const users = await Database.sequelize.query(`SELECT u.id, u.username, u.extraData, json_group_array(value) AS seriesIdsToRemove FROM users u, json_each(u.extraData->"seriesHideFromContinueListening") LEFT JOIN series se ON se.id = value WHERE se.id IS NULL GROUP BY u.id;`, { - model: Database.userModel, - type: Sequelize.QueryTypes.SELECT + const users = await Database.userModel.findAll({ + attributes: ['id', 'username', 'extraData'] }) + for (const user of users) { - const extraData = JSON.parse(user.extraData) - const existingSeriesIds = extraData.seriesHideFromContinueListening - const seriesIdsToRemove = JSON.parse(user.dataValues.seriesIdsToRemove) + const extraData = typeof user.extraData === 'string' ? JSON.parse(user.extraData || '{}') : user.extraData || {} + const existingSeriesIds = Array.isArray(extraData.seriesHideFromContinueListening) ? extraData.seriesHideFromContinueListening : [] + if (!existingSeriesIds.length) continue + + const existingSeries = await Database.seriesModel.findAll({ + attributes: ['id'], + where: { + id: { + [Sequelize.Op.in]: existingSeriesIds + } + } + }) + const existingSeriesSet = new Set(existingSeries.map((series) => series.id)) + const seriesIdsToRemove = existingSeriesIds.filter((seriesId) => !existingSeriesSet.has(seriesId)) + if (!seriesIdsToRemove.length) continue + Logger.info(`[Server] Found ${seriesIdsToRemove.length} non-existent series in seriesHideFromContinueListening for user "${user.username}" - Removing (${seriesIdsToRemove.join(',')})`) const newExtraData = { ...extraData, diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 71f210106..7c7bab873 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -24,6 +24,7 @@ const libraryFilters = require('../utils/queries/libraryFilters') const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters') const authorFilters = require('../utils/queries/authorFilters') const zipHelpers = require('../utils/zipHelpers') +const { isPostgres, noCaseSortExpression, jsonArrayExpand } = require('../utils/sqlDialectHelpers') /** * @typedef RequestUserObject @@ -1039,9 +1040,9 @@ class LibraryController { let order = undefined const direction = payload.sortDesc ? 'DESC' : 'ASC' if (payload.sortBy === 'name') { - order = [[Sequelize.literal('name COLLATE NOCASE'), direction]] + order = [[Sequelize.literal(noCaseSortExpression('name', Database.sequelize)), direction]] } else if (payload.sortBy === 'lastFirst') { - order = [[Sequelize.literal('lastFirst COLLATE NOCASE'), direction]] + order = [[Sequelize.literal(noCaseSortExpression('lastFirst', Database.sequelize)), direction]] } else if (payload.sortBy === 'addedAt') { order = [['createdAt', direction]] } else if (payload.sortBy === 'updatedAt') { @@ -1338,16 +1339,23 @@ class LibraryController { const fileExt = req.query.ext === 'abs' ? 'abs' : 'json' const metadataFilename = `metadata.${fileExt}` + const metadataFilenameCountQuery = isPostgres(Database.sequelize) + ? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{metadata,filename}' = :metadataFilename)` + : `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = :metadataFilename)` + const libraryItemsWithMetadata = await Database.libraryItemModel.findAll({ attributes: ['id', 'libraryFiles'], where: [ { libraryId: req.library.id }, - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = "${metadataFilename}")`), { + Sequelize.where(Sequelize.literal(metadataFilenameCountQuery), { [Sequelize.Op.gte]: 1 }) - ] + ], + replacements: { + metadataFilename + } }) if (!libraryItemsWithMetadata.length) { Logger.info(`[LibraryController] No ${metadataFilename} files found to remove`) diff --git a/server/controllers/MeController.js b/server/controllers/MeController.js index ca600d373..d9a922aa1 100644 --- a/server/controllers/MeController.js +++ b/server/controllers/MeController.js @@ -155,21 +155,23 @@ class MeController { * @param {Response} res */ async getListeningSessions(req, res) { - const listeningSessions = await this.getUserListeningSessionsHelper(req.user.id) - + const startedAt = Date.now() const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10 const page = toNumber(req.query.page, 0) - const start = page * itemsPerPage - const sessions = listeningSessions.slice(start, start + itemsPerPage) + Logger.debug( + `[MeController] /api/me/listening-sessions user="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} start` + ) - const payload = { - total: listeningSessions.length, - numPages: Math.ceil(listeningSessions.length / itemsPerPage), + const payload = await this.getUserListeningSessionsPageHelper( + req.user.id, page, - itemsPerPage, - sessions - } + itemsPerPage + ) + + Logger.debug( + `[MeController] /api/me/listening-sessions user="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms` + ) res.json(payload) } @@ -183,6 +185,7 @@ class MeController { * @param {Response} res */ async getItemListeningSessions(req, res) { + const startedAt = Date.now() const libraryItem = await Database.libraryItemModel.getExpandedById(req.params.libraryItemId) const episode = await Database.podcastEpisodeModel.findByPk(req.params.episodeId) @@ -198,21 +201,23 @@ class MeController { } const mediaItemId = episode?.id || libraryItem.mediaId - let listeningSessions = await this.getUserItemListeningSessionsHelper(req.user.id, mediaItemId) - const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10 const page = toNumber(req.query.page, 0) - const start = page * itemsPerPage - const sessions = listeningSessions.slice(start, start + itemsPerPage) + Logger.debug( + `[MeController] /api/me/item/listening-sessions user="${req.user.id}" libraryItem="${req.params.libraryItemId}" episode="${req.params.episodeId || ''}" page=${page} itemsPerPage=${itemsPerPage} start` + ) - const payload = { - total: listeningSessions.length, - numPages: Math.ceil(listeningSessions.length / itemsPerPage), + const payload = await this.getUserListeningSessionsPageHelper( + req.user.id, page, itemsPerPage, - sessions - } + mediaItemId + ) + + Logger.debug( + `[MeController] /api/me/item/listening-sessions user="${req.user.id}" libraryItem="${req.params.libraryItemId}" episode="${req.params.episodeId || ''}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms` + ) res.json(payload) } @@ -226,7 +231,16 @@ class MeController { * @param {Response} res */ async getListeningStats(req, res) { - const listeningStats = await this.getUserListeningStatsHelpers(req.user.id) + const startedAt = Date.now() + const minified = req.query.minified === '1' || req.query.minified === 'true' + Logger.debug(`[MeController] /api/me/listening-stats user="${req.user.id}" start`) + const listeningStats = await this.getUserListeningStatsHelpers(req.user.id, { + includeItems: !minified, + includeRecentSessions: !minified + }) + Logger.debug( + `[MeController] /api/me/listening-stats user="${req.user.id}" minified=${minified} totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms` + ) res.json(listeningStats) } diff --git a/server/controllers/UserController.js b/server/controllers/UserController.js index 0a476a9d6..96dff644d 100644 --- a/server/controllers/UserController.js +++ b/server/controllers/UserController.js @@ -432,14 +432,20 @@ class UserController { * @param {Response} res */ async getListeningSessions(req, res) { - var listeningSessions = await this.getUserListeningSessionsHelper(req.params.id) - + const startedAt = Date.now() const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10 const page = toNumber(req.query.page, 0) + Logger.debug( + `[UserController] /api/users/${req.params.id}/listening-sessions reqUser="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} start` + ) + const payload = await this.getUserListeningSessionsPageHelper( + req.params.id, + page, + itemsPerPage + ) - const start = page * itemsPerPage // Map user to sessions to match the format of the sessions endpoint - const sessions = listeningSessions.slice(start, start + itemsPerPage).map((session) => { + payload.sessions = payload.sessions.map((session) => { return { ...session, user: { @@ -449,13 +455,9 @@ class UserController { } }) - const payload = { - total: listeningSessions.length, - numPages: Math.ceil(listeningSessions.length / itemsPerPage), - page, - itemsPerPage, - sessions - } + Logger.debug( + `[UserController] /api/users/${req.params.id}/listening-sessions reqUser="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms` + ) res.json(payload) } @@ -469,7 +471,18 @@ class UserController { * @param {Response} res */ async getListeningStats(req, res) { - var listeningStats = await this.getUserListeningStatsHelpers(req.params.id) + const startedAt = Date.now() + const minified = req.query.minified === '1' || req.query.minified === 'true' + Logger.debug( + `[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" start` + ) + var listeningStats = await this.getUserListeningStatsHelpers(req.params.id, { + includeItems: !minified, + includeRecentSessions: !minified + }) + Logger.debug( + `[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" minified=${minified} totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms` + ) res.json(listeningStats) } diff --git a/server/managers/BackupManager.js b/server/managers/BackupManager.js index a7b531e62..277982193 100644 --- a/server/managers/BackupManager.js +++ b/server/managers/BackupManager.js @@ -1,3 +1,4 @@ +const childProcess = require('child_process') const sqlite3 = require('sqlite3') const Path = require('path') const Logger = require('../Logger') @@ -47,6 +48,30 @@ class BackupManager { return global.ServerSettings.maxBackupSize || Infinity } + get databaseBackupConfig() { + if (Database.isPostgresDialect()) { + return { + dialect: 'postgres', + entryName: 'absdatabase.postgres.dump' + } + } + + return { + dialect: 'sqlite', + entryName: 'absdatabase.sqlite' + } + } + + getBackupDialect(backup) { + if (backup.key === 'postgres') return 'postgres' + if (backup.key === 'sqlite' || !backup.key) return 'sqlite' + return null + } + + getBackupEntryName(dialect) { + return dialect === 'postgres' ? 'absdatabase.postgres.dump' : 'absdatabase.sqlite' + } + async init() { try { const backupsDirExists = await fs.pathExists(this.backupPath) @@ -130,13 +155,6 @@ class BackupManager { await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err)) return res.status(400).send('Failed to read backup file - backup might not be a valid .zip file') } - if (!entries['absdatabase.sqlite']) { - Logger.error(`[BackupManager] Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.`) - await zip.close().catch(() => {}) - await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err)) - return res.status(500).send('Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.') - } - const detailsEntry = entries['details'] if (!detailsEntry) { Logger.error('[BackupManager] Invalid backup - missing details entry') @@ -151,10 +169,26 @@ class BackupManager { return res.status(400).send('Invalid backup file - details entry too large') } - const data = await zip.entryData('details') - const details = data.toString('utf8').split('\n') + let backup + try { + const data = await zip.entryData('details') + const details = data.toString('utf8').split('\n') + backup = new Backup({ details, fullPath: tempPath }) + } catch (error) { + Logger.error(`[BackupManager] Invalid backup with no readable details file`, tempPath, error) + await zip.close().catch(() => {}) + await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err)) + return res.status(400).send('Invalid backup file. Missing readable details.') + } - const backup = new Backup({ details, fullPath: tempPath }) + const backupDialect = this.getBackupDialect(backup) + const databaseEntryName = this.getBackupEntryName(backupDialect) + if (!backupDialect || !entries[databaseEntryName]) { + Logger.error(`[BackupManager] Invalid backup with no ${databaseEntryName} file - unsupported database backup.`) + await zip.close().catch(() => {}) + await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err)) + return res.status(500).send(`Invalid backup file. Does not include ${databaseEntryName}.`) + } if (!backup.serverVersion) { Logger.error(`[BackupManager] Invalid backup with no server version - might be a backup created before version 2.0.0`) @@ -204,8 +238,24 @@ class BackupManager { const entries = await zip.entries() + const backupDialect = this.getBackupDialect(backup) + const currentDialect = Database.isPostgresDialect() ? 'postgres' : 'sqlite' + if (!backupDialect) { + await zip.close() + return res.status(500).send('Invalid backup file. Unsupported database backup format.') + } + + if (backupDialect !== currentDialect) { + await zip.close() + return res.status(400).send(`Cannot apply a ${backupDialect} backup while using the ${currentDialect} database.`) + } + + if (backupDialect === 'postgres') { + return this.requestApplyPostgresBackup(apiCacheManager, backup, zip, entries, res) + } + // Ensure backup has an absdatabase.sqlite file - if (!Object.keys(entries).includes('absdatabase.sqlite')) { + if (!Object.keys(entries).includes(this.getBackupEntryName(backupDialect))) { Logger.error(`[BackupManager] Cannot apply old backup ${backup.fullPath}`) await zip.close() return res.status(500).send('Invalid backup file. Does not include absdatabase.sqlite. This might be from an older Audiobookshelf server.') @@ -266,6 +316,70 @@ class BackupManager { SocketAuthority.emitter('backup_applied') } + async requestApplyPostgresBackup(apiCacheManager, backup, zip, entries, res) { + const databaseEntryName = this.getBackupEntryName('postgres') + if (!Object.keys(entries).includes(databaseEntryName)) { + Logger.error(`[BackupManager] Cannot apply Postgres backup ${backup.fullPath}`) + await zip.close() + return res.status(500).send(`Invalid backup file. Does not include ${databaseEntryName}.`) + } + + const tempDumpPath = Path.join(global.ConfigPath, 'absdatabase-postgres-temp.dump') + let reconnected = false + let zipClosed = false + + const closeZip = async () => { + if (zipClosed) return + zipClosed = true + await zip.close() + } + + try { + await fs.remove(tempDumpPath) + await zip.extract(databaseEntryName, tempDumpPath) + + if (!(await fs.pathExists(tempDumpPath))) { + await closeZip() + return res.status(500).send('Failed to extract Postgres database dump from backup') + } + + await Database.disconnect() + await this.restorePostgresDb(tempDumpPath) + + await fs.ensureDir(this.ItemsMetadataPath) + await zip.extract('metadata-items/', this.ItemsMetadataPath) + await fs.ensureDir(this.AuthorsMetadataPath) + await zip.extract('metadata-authors/', this.AuthorsMetadataPath) + await closeZip() + + await Database.reconnect() + reconnected = true + + await apiCacheManager.reset() + await CacheManager.purgeAll() + + res.sendStatus(200) + SocketAuthority.emitter('backup_applied') + } catch (error) { + Logger.error(`[BackupManager] Failed to apply Postgres backup`, error) + try { + await closeZip() + } catch (closeError) { + Logger.error(`[BackupManager] Failed to close Postgres backup archive`, closeError) + } + if (!reconnected) { + try { + await Database.reconnect() + } catch (reconnectError) { + Logger.error(`[BackupManager] Failed to reconnect after Postgres backup apply`, reconnectError) + } + } + return res.status(500).send(`Failed to apply Postgres backup: ${error?.message || 'Unknown Error'}`) + } finally { + await fs.remove(tempDumpPath) + } + } + async loadBackups() { try { const filesInDir = await fs.readdir(this.backupPath) @@ -276,7 +390,7 @@ class BackupManager { const fullFilePath = Path.join(this.backupPath, filename) let zip = null - let data = null + let backup = null try { zip = new StreamZip.async({ file: fullFilePath }) const entries = await zip.entries() @@ -293,17 +407,24 @@ class BackupManager { continue } - data = await zip.entryData('details') + const data = await zip.entryData('details') + const details = data.toString('utf8').split('\n') + + backup = new Backup({ details, fullPath: fullFilePath }) + const backupDialect = this.getBackupDialect(backup) + const databaseEntryName = this.getBackupEntryName(backupDialect) + + if (!backupDialect || !entries[databaseEntryName]) { + Logger.error(`[BackupManager] Unsupported database backup format found "${backup.filename}"`) + await zip.close().catch(() => {}) + continue + } } catch (error) { Logger.error(`[BackupManager] Failed to unzip backup "${fullFilePath}"`, error) if (zip) await zip.close().catch(() => {}) continue } - const details = data.toString('utf8').split('\n') - - const backup = new Backup({ details, fullPath: fullFilePath }) - if (!backup.serverVersion) { // Backups before v2 Logger.error(`[BackupManager] Old unsupported backup was found "${backup.filename}"`) @@ -333,33 +454,34 @@ class BackupManager { async runBackup() { // Check if Metadata Path is inside Config Path (otherwise there will be an infinite loop as the archiver tries to zip itself) Logger.info(`[BackupManager] Running Backup`) + const databaseBackupConfig = this.databaseBackupConfig const newBackup = new Backup() - newBackup.setData(this.backupPath) + newBackup.setData(this.backupPath, databaseBackupConfig.dialect) await fs.ensureDir(this.AuthorsMetadataPath) - // Create backup sqlite file - const sqliteBackupPath = await this.backupSqliteDb(newBackup).catch((error) => { - Logger.error(`[BackupManager] Failed to backup sqlite db`, error) + // Create a database dump + const databaseBackupPath = await this.backupDatabase(newBackup).catch((error) => { + Logger.error(`[BackupManager] Failed to backup ${databaseBackupConfig.dialect} database`, error) const errorMsg = error?.message || error || 'Unknown Error' NotificationManager.onBackupFailed(errorMsg) return false }) - if (!sqliteBackupPath) { + if (!databaseBackupPath) { return false } - // Zip sqlite file, /metadata/items, and /metadata/authors folders - const zipResult = await this.zipBackup(sqliteBackupPath, newBackup).catch((error) => { + // Zip database dump, /metadata/items, and /metadata/authors folders + const zipResult = await this.zipBackup(databaseBackupPath, newBackup, databaseBackupConfig.entryName).catch((error) => { Logger.error(`[BackupManager] Backup Failed ${error}`) const errorMsg = error?.message || error || 'Unknown Error' NotificationManager.onBackupFailed(errorMsg) return false }) - // Remove sqlite backup - await fs.remove(sqliteBackupPath) + // Remove temporary database dump + await fs.remove(databaseBackupPath) if (!zipResult) return false @@ -390,6 +512,10 @@ class BackupManager { return true } + backupDatabase(backup) { + return this.databaseBackupConfig.dialect === 'postgres' ? this.backupPostgresDb(backup) : this.backupSqliteDb(backup) + } + async removeBackup(backup) { try { Logger.debug(`[BackupManager] Removing Backup "${backup.fullPath}"`) @@ -406,29 +532,147 @@ class BackupManager { * @param {Backup} backup */ backupSqliteDb(backup) { - const db = new sqlite3.Database(Database.dbPath) const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.sqlite`) return new Promise(async (resolve, reject) => { - const backup = db.backup(dbFilePath) - backup.step(-1) - backup.finish() + let db + let sqliteBackup + let settled = false - // Max time ~2 mins - for (let i = 0; i < 240; i++) { - if (backup.completed) { - return resolve(dbFilePath) - } else if (backup.failed) { - return reject(backup.message || 'Unknown failure reason') + const finish = (error, result) => { + if (settled) return + settled = true + if (db) { + db.close(() => { + if (error) reject(error) + else resolve(result) + }) + } else if (error) { + reject(error) + } else { + resolve(result) } - await new Promise((r) => setTimeout(r, 500)) } - Logger.error(`[BackupManager] Backup sqlite timed out`) - reject('Backup timed out') + const pollBackup = async () => { + // Max time ~2 mins + for (let i = 0; i < 240; i++) { + if (sqliteBackup.completed) { + return finish(null, dbFilePath) + } else if (sqliteBackup.failed) { + return finish(sqliteBackup.message || 'Unknown failure reason') + } + await new Promise((r) => setTimeout(r, 500)) + } + + Logger.error(`[BackupManager] Backup sqlite timed out`) + finish('Backup timed out') + } + + const startBackup = () => { + try { + sqliteBackup = db.backup(dbFilePath) + sqliteBackup.step(-1) + sqliteBackup.finish() + pollBackup().catch(finish) + } catch (error) { + finish(error) + } + } + + db = new sqlite3.Database(Database.dbPath, (error) => { + if (error) return finish(error) + startBackup() + }) + db.on('error', finish) }) } - zipBackup(sqliteBackupPath, backup) { + /** + * Build pg_dump/pg_restore connection arguments from DATABASE_URL without + * exposing credentials in argv. execFile error messages and the host process + * list include argv, so the password is passed via PGPASSWORD env instead. + */ + getPostgresConnection() { + let dbUrl + try { + dbUrl = new URL(Database.dbPath) + } catch (error) { + throw new Error('DATABASE_URL must be a valid postgres connection URI to run backups') + } + + const args = ['--host', dbUrl.hostname, '--dbname', decodeURIComponent(dbUrl.pathname.replace(/^\//, ''))] + if (dbUrl.port) args.push('--port', dbUrl.port) + if (dbUrl.username) args.push('--username', decodeURIComponent(dbUrl.username)) + + // Redact both the percent-encoded and decoded password from any error output + const decodedPassword = dbUrl.password ? decodeURIComponent(dbUrl.password) : null + const secrets = dbUrl.password ? [dbUrl.password, decodedPassword] : [] + const env = decodedPassword ? { ...process.env, PGPASSWORD: decodedPassword } : process.env + + return { args, env, secrets } + } + + backupPostgresDb(backup) { + const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.postgres.dump`) + return this.runPostgresCommand('pg_dump', [ + '--format=custom', + '--no-owner', + '--no-acl', + '--file', + dbFilePath + ]) + .then(() => dbFilePath) + .catch(async (error) => { + await fs.remove(dbFilePath) + throw error + }) + } + + restorePostgresDb(dbFilePath) { + return this.runPostgresCommand('pg_restore', [ + '--clean', + '--if-exists', + '--exit-on-error', + '--single-transaction', + '--no-owner', + '--no-acl', + dbFilePath + ]) + } + + runPostgresCommand(command, args) { + return new Promise((resolve, reject) => { + let connection + try { + connection = this.getPostgresConnection() + } catch (error) { + return reject(error) + } + + const redact = (text) => { + if (typeof text !== 'string') return text + return connection.secrets.reduce((redacted, secret) => redacted.split(secret).join('***'), text) + } + + const options = { + maxBuffer: 10 * 1024 * 1024, + // Kill after 30 mins (e.g. lock waits) - a killed restore rolls back via --single-transaction + timeout: 30 * 60 * 1000, + env: connection.env + } + childProcess.execFile(command, [...args, ...connection.args], options, (error, stdout, stderr) => { + if (error) { + error.message = redact(error.message) + if (error.cmd) error.cmd = redact(error.cmd) + error.stderr = redact(stderr) + return reject(error) + } + resolve({ stdout, stderr }) + }) + }) + } + + zipBackup(databaseBackupPath, backup, databaseEntryName = 'absdatabase.sqlite') { return new Promise((resolve, reject) => { // create a file to stream archive data to const output = fs.createWriteStream(backup.fullPath) @@ -492,7 +736,7 @@ class BackupManager { // pipe archive data to the file archive.pipe(output) - archive.file(sqliteBackupPath, { name: 'absdatabase.sqlite' }) + archive.file(databaseBackupPath, { name: databaseEntryName }) archive.directory(this.ItemsMetadataPath, 'metadata-items') archive.directory(this.AuthorsMetadataPath, 'metadata-authors') diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index e302038ff..99c89e155 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -94,20 +94,26 @@ class MigrationManager { // Only proceed with migration if there are migrations to run if (migrationsToRun.length > 0) { + const dialect = typeof this.sequelize.getDialect === 'function' ? this.sequelize.getDialect() : 'sqlite' + const isSqlite = !dialect || dialect === 'sqlite' const originalDbPath = path.join(this.configPath, 'absdatabase.sqlite') const backupDbPath = path.join(this.configPath, 'absdatabase.backup.sqlite') try { Logger.info(`[MigrationManager] Migrating database ${migrationDirection} to version ${this.serverVersion}`) Logger.info(`[MigrationManager] Migrations to run: ${migrationsToRun.join(', ')}`) - // Create a backup copy of the SQLite database before starting migrations - await fs.copy(originalDbPath, backupDbPath) - Logger.info('Created a backup of the original database.') + if (isSqlite) { + // Create a backup copy of the SQLite database before starting migrations + await fs.copy(originalDbPath, backupDbPath) + Logger.info('Created a backup of the original database.') + } // Run migrations await this.umzug[migrationDirection]({ migrations: migrationsToRun, rerun: 'ALLOW' }) - // Clean up the backup - await fs.remove(backupDbPath) + if (isSqlite) { + // Clean up the backup + await fs.remove(backupDbPath) + } Logger.info('[MigrationManager] Migrations successfully applied to the original database.') } catch (error) { @@ -115,13 +121,15 @@ class MigrationManager { await this.sequelize.close() - // Step 3: If migration fails, save the failed original and restore the backup - const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite') - await fs.move(originalDbPath, failedDbPath, { overwrite: true }) - Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.') + if (isSqlite) { + // Step 3: If migration fails, save the failed original and restore the backup + const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite') + await fs.move(originalDbPath, failedDbPath, { overwrite: true }) + Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.') - await fs.move(backupDbPath, originalDbPath, { overwrite: true }) - Logger.info('[MigrationManager] Restored the original database from the backup.') + await fs.move(backupDbPath, originalDbPath, { overwrite: true }) + Logger.info('[MigrationManager] Restored the original database from the backup.') + } Logger.info('[MigrationManager] Migration failed. Exiting Audiobookshelf with code 1.') process.exit(1) @@ -191,29 +199,27 @@ class MigrationManager { } async fetchVersionsFromDatabase() { + const migrationsMetaTable = MigrationManager.MIGRATIONS_META_TABLE await this.checkOrCreateMigrationsMetaTable() - const [{ version }] = await this.sequelize.query("SELECT value as version FROM :migrationsMeta WHERE key = 'version'", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [versionRow] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, { type: Sequelize.QueryTypes.SELECT }) - this.databaseVersion = version + this.databaseVersion = versionRow?.version - const [{ maxVersion }] = await this.sequelize.query("SELECT value as maxVersion FROM :migrationsMeta WHERE key = 'maxVersion'", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [maxVersionRow] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, { type: Sequelize.QueryTypes.SELECT }) - this.maxVersion = maxVersion + this.maxVersion = maxVersionRow?.maxVersion || maxVersionRow?.maxversion } async checkOrCreateMigrationsMetaTable() { const queryInterface = this.sequelize.getQueryInterface() - let migrationsMetaTableExists = await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE) + let migrationsMetaTableExists = await this.tableExists(MigrationManager.MIGRATIONS_META_TABLE) // If the table exists, check that the `version` and `maxVersion` rows exist if (migrationsMetaTableExists) { - const [{ count }] = await this.sequelize.query("SELECT COUNT(*) as count FROM :migrationsMeta WHERE key IN ('version', 'maxVersion')", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM ${MigrationManager.MIGRATIONS_META_TABLE} WHERE key IN ('version', 'maxVersion')`, { type: Sequelize.QueryTypes.SELECT }) if (count < 2) { @@ -241,14 +247,28 @@ class MigrationManager { allowNull: false } }) - await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", { - replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`INSERT INTO ${MigrationManager.MIGRATIONS_META_TABLE} (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, { + replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0' }, type: Sequelize.QueryTypes.INSERT }) Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`) } } + async tableExists(tableName) { + const queryInterface = this.sequelize.getQueryInterface() + if (typeof queryInterface.tableExists === 'function') { + return queryInterface.tableExists(tableName) + } + + const tables = await queryInterface.showAllTables() + return tables.some((table) => { + if (typeof table === 'string') return table === tableName + if (table?.tableName) return table.tableName === tableName + return false + }) + } + extractVersionFromTag(tag) { if (!tag) return null const versionMatch = tag.match(/^v?(\d+\.\d+\.\d+)/) @@ -299,8 +319,8 @@ class MigrationManager { async updateMaxVersion() { try { - await this.sequelize.query("UPDATE :migrationsMeta SET value = :maxVersion WHERE key = 'maxVersion'", { - replacements: { maxVersion: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :maxVersion WHERE key = 'maxVersion'`, { + replacements: { maxVersion: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) } catch (error) { @@ -311,8 +331,8 @@ class MigrationManager { async updateDatabaseVersion() { try { - await this.sequelize.query("UPDATE :migrationsMeta SET value = :version WHERE key = 'version'", { - replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :version WHERE key = 'version'`, { + replacements: { version: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) } catch (error) { diff --git a/server/migrations/v2.15.0-series-column-unique.js b/server/migrations/v2.15.0-series-column-unique.js index 7f8526f97..883ce09fc 100644 --- a/server/migrations/v2.15.0-series-column-unique.js +++ b/server/migrations/v2.15.0-series-column-unique.js @@ -19,12 +19,21 @@ async function up({ context: { queryInterface, logger } }) { logger.info('[2.15.0 migration] UPGRADE BEGIN: 2.15.0-series-column-unique ') // Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0 - logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') - await queryInterface.sequelize.query('REINDEX NOCASE;') + if (queryInterface.sequelize.getDialect() === 'sqlite') { + logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') + await queryInterface.sequelize.query('REINDEX NOCASE;') + } else { + logger.info('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect') + } // Check if the unique index already exists const seriesIndexes = await queryInterface.showIndex('Series') - if (seriesIndexes.some((index) => index.name === 'unique_series_name_per_library')) { + if ( + seriesIndexes.some((index) => { + const indexName = index?.name || index?.indexName || '' + return String(indexName).toLowerCase() === 'unique_series_name_per_library' + }) + ) { logger.info('[2.15.0 migration] Unique index on Series.name and Series.libraryId already exists') logger.info('[2.15.0 migration] UPGRADE END: 2.15.0-series-column-unique ') return @@ -181,11 +190,20 @@ async function up({ context: { queryInterface, logger } }) { logger.info(`[2.15.0 migration] Deduplication complete`) // Create a unique index based on the name and library ID for the `Series` table - await queryInterface.addIndex('Series', ['name', 'libraryId'], { - unique: true, - name: 'unique_series_name_per_library' - }) - logger.info('[2.15.0 migration] Added unique index on Series.name and Series.libraryId') + try { + await queryInterface.addIndex('Series', ['name', 'libraryId'], { + unique: true, + name: 'unique_series_name_per_library' + }) + logger.info('[2.15.0 migration] Added unique index on Series.name and Series.libraryId') + } catch (error) { + const alreadyExists = + (error?.name === 'SequelizeDatabaseError' && /already exists/i.test(error?.message || '')) || + error?.original?.code === '42P07' + if (!alreadyExists) throw error + + logger.info('[2.15.0 migration] Unique index on Series.name and Series.libraryId already exists') + } logger.info('[2.15.0 migration] UPGRADE END: 2.15.0-series-column-unique ') } diff --git a/server/migrations/v2.15.1-reindex-nocase.js b/server/migrations/v2.15.1-reindex-nocase.js index 2ec9487d2..429edf9ff 100644 --- a/server/migrations/v2.15.1-reindex-nocase.js +++ b/server/migrations/v2.15.1-reindex-nocase.js @@ -17,6 +17,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info('[2.15.1 migration] UPGRADE BEGIN: 2.15.1-reindex-nocase ') + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect') + logger.info('[2.15.1 migration] UPGRADE END: 2.15.1-reindex-nocase ') + return + } + // Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0 logger.info('[2.15.1 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') await queryInterface.sequelize.query('REINDEX NOCASE;') diff --git a/server/migrations/v2.15.2-index-creation.js b/server/migrations/v2.15.2-index-creation.js index f1302dd26..03113e8df 100644 --- a/server/migrations/v2.15.2-index-creation.js +++ b/server/migrations/v2.15.2-index-creation.js @@ -20,10 +20,8 @@ async function up({ context: { queryInterface, logger } }) { // Create index for bookAuthors logger.info('[2.15.2 migration] Creating index for bookAuthors') const bookAuthorsIndexes = await queryInterface.showIndex('bookAuthors') - if (!bookAuthorsIndexes.some((index) => index.name === 'bookAuthor_authorId')) { - await queryInterface.addIndex('bookAuthors', ['authorId'], { - name: 'bookAuthor_authorId' - }) + if (!hasIndex(bookAuthorsIndexes, 'bookAuthor_authorId')) { + await addIndexIfMissing(queryInterface, 'bookAuthors', ['authorId'], 'bookAuthor_authorId', logger) } else { logger.info('[2.15.2 migration] Index bookAuthor_authorId already exists') } @@ -31,25 +29,21 @@ async function up({ context: { queryInterface, logger } }) { // Create index for bookSeries logger.info('[2.15.2 migration] Creating index for bookSeries') const bookSeriesIndexes = await queryInterface.showIndex('bookSeries') - if (!bookSeriesIndexes.some((index) => index.name === 'bookSeries_seriesId')) { - await queryInterface.addIndex('bookSeries', ['seriesId'], { - name: 'bookSeries_seriesId' - }) + if (!hasIndex(bookSeriesIndexes, 'bookSeries_seriesId')) { + await addIndexIfMissing(queryInterface, 'bookSeries', ['seriesId'], 'bookSeries_seriesId', logger) } else { logger.info('[2.15.2 migration] Index bookSeries_seriesId already exists') } // Delete existing podcastEpisode index logger.info('[2.15.2 migration] Deleting existing podcastEpisode index') - await queryInterface.removeIndex('podcastEpisodes', 'podcast_episodes_created_at') + await removeIndexIfExists(queryInterface, 'podcastEpisodes', 'podcast_episodes_created_at', logger) // Create index for podcastEpisode and createdAt logger.info('[2.15.2 migration] Creating index for podcastEpisode and createdAt') const podcastEpisodesIndexes = await queryInterface.showIndex('podcastEpisodes') - if (!podcastEpisodesIndexes.some((index) => index.name === 'podcastEpisode_createdAt_podcastId')) { - await queryInterface.addIndex('podcastEpisodes', ['createdAt', 'podcastId'], { - name: 'podcastEpisode_createdAt_podcastId' - }) + if (!hasIndex(podcastEpisodesIndexes, 'podcastEpisode_createdAt_podcastId')) { + await addIndexIfMissing(queryInterface, 'podcastEpisodes', ['createdAt', 'podcastId'], 'podcastEpisode_createdAt_podcastId', logger) } else { logger.info('[2.15.2 migration] Index podcastEpisode_createdAt_podcastId already exists') } @@ -78,7 +72,7 @@ async function down({ context: { queryInterface, logger } }) { // Delete existing podcastEpisode index logger.info('[2.15.2 migration] Deleting existing podcastEpisode index') - await queryInterface.removeIndex('podcastEpisodes', 'podcastEpisode_createdAt_podcastId') + await removeIndexIfExists(queryInterface, 'podcastEpisodes', 'podcastEpisode_createdAt_podcastId', logger) // Create index for podcastEpisode and createdAt logger.info('[2.15.2 migration] Creating original index for podcastEpisode createdAt') @@ -91,3 +85,33 @@ async function down({ context: { queryInterface, logger } }) { } module.exports = { up, down } + +async function removeIndexIfExists(queryInterface, tableName, indexName, logger) { + const indexes = await queryInterface.showIndex(tableName) + const hasIndexWithName = hasIndex(indexes, indexName) + + if (!hasIndexWithName) { + logger.info(`[2.15.2 migration] Index ${indexName} does not exist, skipping removeIndex`) + return + } + + await queryInterface.removeIndex(tableName, indexName) +} + +function hasIndex(indexes, indexName) { + const expected = String(indexName || '').toLowerCase() + return indexes.some((index) => String(index?.name || index?.indexName || '').toLowerCase() === expected) +} + +async function addIndexIfMissing(queryInterface, tableName, fields, indexName, logger) { + try { + await queryInterface.addIndex(tableName, fields, { name: indexName }) + } catch (error) { + const alreadyExists = + (error?.name === 'SequelizeDatabaseError' && /already exists/i.test(error?.message || '')) || + error?.original?.code === '42P07' + if (!alreadyExists) throw error + + logger.info(`[2.15.2 migration] Index ${indexName} already exists`) + } +} diff --git a/server/migrations/v2.17.3-fk-constraints.js b/server/migrations/v2.17.3-fk-constraints.js index 5f8a5c9a6..3ad61e585 100644 --- a/server/migrations/v2.17.3-fk-constraints.js +++ b/server/migrations/v2.17.3-fk-constraints.js @@ -18,6 +18,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info('[2.17.3 migration] UPGRADE BEGIN: 2.17.3-fk-constraints') + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect') + logger.info('[2.17.3 migration] UPGRADE END: 2.17.3-fk-constraints') + return + } + const execQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize) // Disable foreign key constraints for the next sequence of operations diff --git a/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.js b/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.js index 03797e35e..08ab3b983 100644 --- a/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.js +++ b/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.js @@ -56,15 +56,23 @@ async function down({ context: { queryInterface, logger } }) { } async function getServerSettings(queryInterface, logger) { - const result = await queryInterface.sequelize.query('SELECT value FROM settings WHERE key = "server-settings";') + const result = await queryInterface.sequelize.query('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) if (!result[0].length) { logger.error('[2.17.4 migration] Server settings not found') throw new Error('Server settings not found') } + const settingsValue = result[0][0].value + + if (settingsValue && typeof settingsValue === 'object') { + return settingsValue + } + let serverSettings = null try { - serverSettings = JSON.parse(result[0][0].value) + serverSettings = JSON.parse(settingsValue) } catch (error) { logger.error('[2.17.4 migration] Error parsing server settings:', error) throw error @@ -74,9 +82,10 @@ async function getServerSettings(queryInterface, logger) { } async function updateServerSettings(queryInterface, logger, serverSettings) { - await queryInterface.sequelize.query('UPDATE settings SET value = :value WHERE key = "server-settings";', { + await queryInterface.sequelize.query('UPDATE settings SET value = :value WHERE key = :settingsKey;', { replacements: { - value: JSON.stringify(serverSettings) + value: JSON.stringify(serverSettings), + settingsKey: 'server-settings' } }) } diff --git a/server/migrations/v2.19.1-copy-title-to-library-items.js b/server/migrations/v2.19.1-copy-title-to-library-items.js index 7b75fa855..d62b008ed 100644 --- a/server/migrations/v2.19.1-copy-title-to-library-items.js +++ b/server/migrations/v2.19.1-copy-title-to-library-items.js @@ -25,6 +25,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`) + logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`) + return + } + await addColumn(queryInterface, logger, 'libraryItems', 'title', { type: queryInterface.sequelize.Sequelize.STRING, allowNull: true }) await copyColumn(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId') await addTrigger(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId') @@ -51,6 +57,12 @@ async function down({ context: { queryInterface, logger } }) { // Downward migration script logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`) + logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`) + return + } + await removeIndex(queryInterface, logger, 'libraryItems', ['libraryId', 'mediaType', 'title']) await removeTrigger(queryInterface, logger, 'libraryItems', 'title') await removeColumn(queryInterface, logger, 'libraryItems', 'title') diff --git a/server/migrations/v2.19.4-improve-podcast-queries.js b/server/migrations/v2.19.4-improve-podcast-queries.js index 689795c31..3becd3162 100644 --- a/server/migrations/v2.19.4-improve-podcast-queries.js +++ b/server/migrations/v2.19.4-improve-podcast-queries.js @@ -26,6 +26,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`) + logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`) + return + } + // Add numEpisodes column to podcasts table await addColumn(queryInterface, logger, 'podcasts', 'numEpisodes', { type: queryInterface.sequelize.Sequelize.INTEGER, allowNull: false, defaultValue: 0 }) @@ -60,6 +66,12 @@ async function down({ context: { queryInterface, logger } }) { // Downward migration script logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`) + logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`) + return + } + // Remove triggers from libraryItems await removeTrigger(queryInterface, logger, 'podcasts', 'title', 'libraryItems', 'title') await removeTrigger(queryInterface, logger, 'podcasts', 'titleIgnorePrefix', 'libraryItems', 'titleIgnorePrefix') diff --git a/server/migrations/v2.20.0-improve-author-sort-queries.js b/server/migrations/v2.20.0-improve-author-sort-queries.js index 53016c709..c5c2b8f5a 100644 --- a/server/migrations/v2.20.0-improve-author-sort-queries.js +++ b/server/migrations/v2.20.0-improve-author-sort-queries.js @@ -39,6 +39,11 @@ const authorsJoin = `${authors} JOIN ${bookAuthors} ON ${authors}.id = ${bookAut * @returns {Promise} - A promise that resolves when the migration is complete. */ async function up({ context: { queryInterface, logger } }) { + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`) + return + } + const helper = new MigrationHelper(queryInterface, logger) // Upwards migration script @@ -72,6 +77,11 @@ async function up({ context: { queryInterface, logger } }) { * @returns {Promise} - A promise that resolves when the migration is complete. */ async function down({ context: { queryInterface, logger } }) { + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`) + return + } + // Downward migration script logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) diff --git a/server/migrations/v2.33.0-add-discover-query-indexes.js b/server/migrations/v2.33.0-add-discover-query-indexes.js index ebd92bbaa..8500a5c18 100644 --- a/server/migrations/v2.33.0-add-discover-query-indexes.js +++ b/server/migrations/v2.33.0-add-discover-query-indexes.js @@ -45,8 +45,8 @@ async function down({ context: { queryInterface, logger } }) { } async function addIndexIfMissing(queryInterface, logger, index) { - const existing = await queryInterface.showIndex(index.table) - if (existing.some((i) => i.name === index.name)) { + const existing = await showIndexNames(queryInterface, index.table) + if (existing.some((name) => name.toLowerCase() === index.name.toLowerCase())) { logger.info(`${loggerPrefix} index ${index.name} already exists on ${index.table}`) return } @@ -60,8 +60,8 @@ async function addIndexIfMissing(queryInterface, logger, index) { } async function removeIndexIfExists(queryInterface, logger, index) { - const existing = await queryInterface.showIndex(index.table) - if (!existing.some((i) => i.name === index.name)) { + const existing = await showIndexNames(queryInterface, index.table) + if (!existing.some((name) => name.toLowerCase() === index.name.toLowerCase())) { logger.info(`${loggerPrefix} index ${index.name} does not exist on ${index.table}`) return } @@ -71,4 +71,21 @@ async function removeIndexIfExists(queryInterface, logger, index) { logger.info(`${loggerPrefix} removed index ${index.name}`) } +/** + * Sequelize showIndex matches the table name case-sensitively, but postgres folds + * unquoted identifiers to lowercase, so query pg_indexes directly on postgres. + * + * @returns {Promise} index names on the table + */ +async function showIndexNames(queryInterface, table) { + if (queryInterface.sequelize.getDialect() === 'postgres') { + const [rows] = await queryInterface.sequelize.query('SELECT indexname AS name FROM pg_indexes WHERE schemaname = current_schema() AND tablename = $1', { + bind: [table.toLowerCase()] + }) + return rows.map((row) => row.name) + } + const existing = await queryInterface.showIndex(table) + return existing.map((row) => row.name) +} + module.exports = { up, down } diff --git a/server/migrations/v2.35.0-add-last-refresh-token.js b/server/migrations/v2.35.0-add-last-refresh-token.js index 0ad190e9a..a689c2914 100644 --- a/server/migrations/v2.35.0-add-last-refresh-token.js +++ b/server/migrations/v2.35.0-add-last-refresh-token.js @@ -22,8 +22,10 @@ async function up({ context: { queryInterface, logger } }) { if (await queryInterface.tableExists('sessions')) { const tableDescription = await queryInterface.describeTable('sessions') + // Postgres folds unquoted identifiers to lowercase, so match column names case-insensitively + const hasColumn = (name) => Object.keys(tableDescription).some((column) => column.toLowerCase() === name.toLowerCase()) - if (!tableDescription.lastRefreshToken) { + if (!hasColumn('lastRefreshToken')) { logger.info(`${loggerPrefix} Adding lastRefreshToken column to sessions table`) await queryInterface.addColumn('sessions', 'lastRefreshToken', { type: queryInterface.sequelize.Sequelize.DataTypes.STRING, @@ -33,7 +35,7 @@ async function up({ context: { queryInterface, logger } }) { logger.info(`${loggerPrefix} lastRefreshToken column already exists in sessions table`) } - if (!tableDescription.lastRefreshTokenExpiresAt) { + if (!hasColumn('lastRefreshTokenExpiresAt')) { logger.info(`${loggerPrefix} Adding lastRefreshTokenExpiresAt column to sessions table`) await queryInterface.addColumn('sessions', 'lastRefreshTokenExpiresAt', { type: queryInterface.sequelize.Sequelize.DataTypes.DATE, @@ -60,15 +62,17 @@ async function down({ context: { queryInterface, logger } }) { if (await queryInterface.tableExists('sessions')) { const tableDescription = await queryInterface.describeTable('sessions') + // Postgres folds unquoted identifiers to lowercase, so match column names case-insensitively + const hasColumn = (name) => Object.keys(tableDescription).some((column) => column.toLowerCase() === name.toLowerCase()) - if (tableDescription.lastRefreshToken) { + if (hasColumn('lastRefreshToken')) { logger.info(`${loggerPrefix} Removing lastRefreshToken column from sessions table`) await queryInterface.removeColumn('sessions', 'lastRefreshToken') } else { logger.info(`${loggerPrefix} lastRefreshToken column does not exist in sessions table`) } - if (tableDescription.lastRefreshTokenExpiresAt) { + if (hasColumn('lastRefreshTokenExpiresAt')) { logger.info(`${loggerPrefix} Removing lastRefreshTokenExpiresAt column from sessions table`) await queryInterface.removeColumn('sessions', 'lastRefreshTokenExpiresAt') } else { diff --git a/server/models/Author.js b/server/models/Author.js index 65561e211..f9a670351 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -129,6 +129,8 @@ class Author extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' } + super.init( { id: { @@ -147,12 +149,7 @@ class Author extends Model { modelName: 'author', indexes: [ { - fields: [ - { - name: 'name', - collate: 'NOCASE' - } - ] + fields: [nameIndexField] }, // { // fields: [{ diff --git a/server/models/Book.js b/server/models/Book.js index 7e1a2e30c..558950072 100644 --- a/server/models/Book.js +++ b/server/models/Book.js @@ -138,6 +138,8 @@ class Book extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' } + super.init( { id: { @@ -147,7 +149,7 @@ class Book extends Model { }, title: DataTypes.STRING, titleIgnorePrefix: DataTypes.STRING, - subtitle: DataTypes.STRING, + subtitle: DataTypes.TEXT, publishedYear: DataTypes.STRING, publishedDate: DataTypes.STRING, publisher: DataTypes.STRING, @@ -172,12 +174,7 @@ class Book extends Model { modelName: 'book', indexes: [ { - fields: [ - { - name: 'title', - collate: 'NOCASE' - } - ] + fields: [titleIndexField] }, // { // fields: [{ diff --git a/server/models/LibraryItem.js b/server/models/LibraryItem.js index c03667efc..6c464502e 100644 --- a/server/models/LibraryItem.js +++ b/server/models/LibraryItem.js @@ -720,6 +720,11 @@ class LibraryItem extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' } + const titleIgnorePrefixIndexField = sequelize.getDialect() === 'postgres' ? 'titleIgnorePrefix' : { name: 'titleIgnorePrefix', collate: 'NOCASE' } + const authorNamesFirstLastIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesFirstLast' : { name: 'authorNamesFirstLast', collate: 'NOCASE' } + const authorNamesLastFirstIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesLastFirst' : { name: 'authorNamesLastFirst', collate: 'NOCASE' } + super.init( { id: { @@ -768,16 +773,16 @@ class LibraryItem extends Model { fields: ['libraryId', 'mediaType', 'createdAt'] }, { - fields: ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', titleIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', titleIgnorePrefixIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', authorNamesFirstLastIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', authorNamesLastFirstIndexField] }, { fields: ['libraryId', 'mediaId', 'mediaType'] diff --git a/server/models/MediaProgress.js b/server/models/MediaProgress.js index 9c0269a9e..e562b7109 100644 --- a/server/models/MediaProgress.js +++ b/server/models/MediaProgress.js @@ -257,7 +257,13 @@ class MediaProgress extends Model { const escapedDate = this.sequelize.escape(new Date(progressPayload.lastUpdate)) Logger.info(`[MediaProgress] Manually setting updatedAt to ${escapedDate} (media item ${this.mediaItemId})`) - await this.sequelize.query(`UPDATE "mediaProgresses" SET "updatedAt" = ${escapedDate} WHERE "id" = '${this.id}'`) + await this.constructor.update( + { updatedAt: new Date(progressPayload.lastUpdate) }, + { + where: { id: this.id }, + silent: true + } + ) await this.reload() } diff --git a/server/models/PlaybackSession.js b/server/models/PlaybackSession.js index 196fbda6c..26b702d8c 100644 --- a/server/models/PlaybackSession.js +++ b/server/models/PlaybackSession.js @@ -52,18 +52,54 @@ class PlaybackSession extends Model { this.createdAt } - static async getOldPlaybackSessions(where = null) { + static async getOldPlaybackSessions(where = null, options = {}) { + const { + limit = null, + offset = null, + order = [['updatedAt', 'DESC']], + includeDevice = true + } = options + const playbackSessions = await this.findAll({ where, - include: [ - { - model: this.sequelize.models.device - } - ] + limit, + offset, + order, + include: includeDevice + ? [ + { + model: this.sequelize.models.device + } + ] + : undefined }) return playbackSessions.map((session) => this.getOldPlaybackSession(session)) } + static countWithWhere(where = null) { + return this.count({ where }) + } + + static getPlaybackSessionsForStats(where = null) { + return this.findAll({ + where, + order: [['updatedAt', 'DESC']], + attributes: [ + 'mediaItemId', + 'mediaItemType', + 'displayTitle', + 'displayAuthor', + 'timeListening', + 'mediaMetadata', + 'date', + 'dayOfWeek', + 'updatedAt', + 'extraData' + ], + raw: true + }) + } + static async getById(sessionId) { const playbackSession = await this.findByPk(sessionId, { include: [ @@ -190,7 +226,7 @@ class PlaybackSession extends Model { currentTime: DataTypes.FLOAT, serverVersion: DataTypes.STRING, coverPath: DataTypes.STRING, - timeListening: DataTypes.INTEGER, + timeListening: DataTypes.FLOAT, mediaMetadata: DataTypes.JSON, date: DataTypes.STRING, dayOfWeek: DataTypes.STRING, @@ -237,18 +273,20 @@ class PlaybackSession extends Model { if (!Array.isArray(findResult)) findResult = [findResult] for (const instance of findResult) { + const values = instance?.dataValues + if (instance.mediaItemType === 'book' && instance.book !== undefined) { instance.mediaItem = instance.book - instance.dataValues.mediaItem = instance.dataValues.book + if (values) values.mediaItem = values.book } else if (instance.mediaItemType === 'podcastEpisode' && instance.podcastEpisode !== undefined) { instance.mediaItem = instance.podcastEpisode - instance.dataValues.mediaItem = instance.dataValues.podcastEpisode + if (values) values.mediaItem = values.podcastEpisode } // To prevent mistakes: delete instance.book - delete instance.dataValues.book + if (values) delete values.book delete instance.podcastEpisode - delete instance.dataValues.podcastEpisode + if (values) delete values.podcastEpisode } }) } diff --git a/server/models/Series.js b/server/models/Series.js index 6ca288464..192378b7d 100644 --- a/server/models/Series.js +++ b/server/models/Series.js @@ -1,6 +1,7 @@ const { DataTypes, Model, where, fn, col, literal } = require('sequelize') const { getTitlePrefixAtEnd, getTitleIgnorePrefix } = require('../utils/index') +const { safeTextToDoubleExpression } = require('../utils/sqlDialectHelpers') class Series extends Model { constructor(values, options) { @@ -87,6 +88,8 @@ class Series extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' } + super.init( { id: { @@ -103,12 +106,7 @@ class Series extends Model { modelName: 'series', indexes: [ { - fields: [ - { - name: 'name', - collate: 'NOCASE' - } - ] + fields: [nameIndexField] }, // { // fields: [{ @@ -161,7 +159,7 @@ class Series extends Model { } } ], - order: [[literal('CAST(`bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]] + order: [[literal(`${safeTextToDoubleExpression(this.sequelize.getDialect() === 'postgres' ? '"bookSeries"."sequence"' : '`bookSeries.sequence`', this.sequelize)} ASC NULLS LAST`)]] }) } diff --git a/server/models/User.js b/server/models/User.js index 0b9d49438..d93ebc02e 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -18,12 +18,16 @@ class UserCache { } getByEmail(email) { - const user = this.cache.find((u) => u.email === email) + if (!email) return null + const normalizedEmail = email.toLowerCase() + const user = this.cache.find((u) => u.email && u.email.toLowerCase() === normalizedEmail) return user } getByUsername(username) { - const user = this.cache.find((u) => u.username === username) + if (!username) return null + const normalizedUsername = username.toLowerCase() + const user = this.cache.find((u) => u.username && u.username.toLowerCase() === normalizedUsername) return user } @@ -348,11 +352,13 @@ class User extends Model { static async getUserByUsername(username) { if (!username) return null - const cachedUser = userCache.getByUsername(username) + const normalizedUsername = username.toLowerCase() + + const cachedUser = userCache.getByUsername(normalizedUsername) if (cachedUser) return cachedUser const user = await this.findOne({ - where: sequelize.where(sequelize.fn('lower', sequelize.col('username')), username.toLowerCase()), + where: sequelize.where(sequelize.fn('LOWER', sequelize.col('username')), normalizedUsername), include: this.sequelize.models.mediaProgress }) @@ -369,11 +375,13 @@ class User extends Model { static async getUserByEmail(email) { if (!email) return null - const cachedUser = userCache.getByEmail(email) + const normalizedEmail = email.toLowerCase() + + const cachedUser = userCache.getByEmail(normalizedEmail) if (cachedUser) return cachedUser const user = await this.findOne({ - where: sequelize.where(sequelize.fn('lower', sequelize.col('email')), email.toLowerCase()), + where: sequelize.where(sequelize.fn('LOWER', sequelize.col('email')), normalizedEmail), include: this.sequelize.models.mediaProgress }) @@ -414,10 +422,24 @@ class User extends Model { const cachedUser = userCache.getById(userId) || userCache.getByOldId(userId) if (cachedUser) return cachedUser + const isUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userId) + if (isUuid) { + const byId = await this.findByPk(userId, { + include: this.sequelize.models.mediaProgress + }) + if (byId) { + userCache.set(byId) + return byId + } + } + + const oldIdMatcher = + this.sequelize.getDialect() === 'postgres' + ? sequelize.where(sequelize.literal(`extradata#>>'{oldUserId}'`), userId) + : { 'extraData.oldUserId': userId } + const user = await this.findOne({ - where: { - [sequelize.Op.or]: [{ id: userId }, { 'extraData.oldUserId': userId }] - }, + where: oldIdMatcher, include: this.sequelize.models.mediaProgress }) diff --git a/server/objects/Backup.js b/server/objects/Backup.js index e3b9f4b4a..e12c294c5 100644 --- a/server/objects/Backup.js +++ b/server/objects/Backup.js @@ -62,9 +62,9 @@ class Backup { } } - setData(backupDirPath) { + setData(backupDirPath, dialect = 'sqlite') { this.id = date.format(new Date(), 'YYYY-MM-DD[T]HHmm') - this.key = 'sqlite' + this.key = dialect this.datePretty = date.format(new Date(), 'ddd, MMM D YYYY HH:mm') this.backupDirPath = backupDirPath @@ -78,4 +78,4 @@ class Backup { this.createdAt = Date.now() } } -module.exports = Backup \ No newline at end of file +module.exports = Backup diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index 7c89126b2..a674a2948 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -527,54 +527,128 @@ class ApiRouter { return userSessions.sort((a, b) => b.updatedAt - a.updatedAt) } + async getUserListeningSessionsPageHelper(userId, page = 0, itemsPerPage = 10, mediaItemId = null) { + const where = { userId } + if (mediaItemId) where.mediaItemId = mediaItemId + + const start = page * itemsPerPage + const [total, sessions] = await Promise.all([ + Database.countPlaybackSessions(where), + Database.getPlaybackSessions(where, { + limit: itemsPerPage, + offset: start, + order: [['updatedAt', 'DESC']] + }) + ]) + + return { + total, + numPages: Math.ceil(total / itemsPerPage), + page, + itemsPerPage, + sessions + } + } + async getUserItemListeningSessionsHelper(userId, mediaItemId) { const userSessions = await Database.getPlaybackSessions({ userId, mediaItemId }) return userSessions.sort((a, b) => b.updatedAt - a.updatedAt) } - async getUserListeningStatsHelpers(userId) { + async getUserListeningStatsHelpers(userId, options = {}) { + const startedAt = Date.now() const today = date.format(new Date(), 'YYYY-MM-DD') + const includeItems = options.includeItems !== false + const includeRecentSessions = options.includeRecentSessions !== false + const getSessionField = (session, key) => { + if (!session || typeof session !== 'object') return undefined + if (session[key] !== undefined) return session[key] + + const lowerKey = key.toLowerCase() + if (session[lowerKey] !== undefined) return session[lowerKey] + + return undefined + } + const getSessionObjectField = (session, key) => { + const value = getSessionField(session, key) + if (!value) return null + if (typeof value === 'string') { + try { + return JSON.parse(value) + } catch (error) { + return null + } + } + return value + } + + const tasks = [Database.getPlaybackSessionsForStats({ userId })] + if (includeRecentSessions) { + tasks.push( + Database.getPlaybackSessions({ userId }, { + limit: 10, + offset: 0, + order: [['updatedAt', 'DESC']] + }) + ) + } + const [listeningSessions, recentSessions = []] = await Promise.all(tasks) - const listeningSessions = await this.getUserListeningSessionsHelper(userId) const listeningStats = { totalTime: 0, - items: {}, days: {}, dayOfWeek: {}, - today: 0, - recentSessions: listeningSessions.slice(0, 10) + today: 0 } + if (includeItems) listeningStats.items = {} + if (includeRecentSessions) listeningStats.recentSessions = recentSessions listeningSessions.forEach((s) => { - let sessionTimeListening = s.timeListening - if (typeof sessionTimeListening == 'string') { - sessionTimeListening = Number(sessionTimeListening) - } + const extraData = getSessionObjectField(s, 'extraData') + const libraryItemId = extraData?.libraryItemId || extraData?.libraryitemid || null + const sessionDate = getSessionField(s, 'date') + const sessionDayOfWeek = getSessionField(s, 'dayOfWeek') + const sessionUpdatedAt = getSessionField(s, 'updatedAt') + const sessionMediaMetadata = getSessionObjectField(s, 'mediaMetadata') + const numericListening = Number(getSessionField(s, 'timeListening')) + const sessionTimeListening = Number.isFinite(numericListening) + ? numericListening + : 0 - if (s.dayOfWeek) { - if (!listeningStats.dayOfWeek[s.dayOfWeek]) listeningStats.dayOfWeek[s.dayOfWeek] = 0 - listeningStats.dayOfWeek[s.dayOfWeek] += sessionTimeListening + if (sessionDayOfWeek) { + if (!listeningStats.dayOfWeek[sessionDayOfWeek]) listeningStats.dayOfWeek[sessionDayOfWeek] = 0 + listeningStats.dayOfWeek[sessionDayOfWeek] += sessionTimeListening } - if (s.date && sessionTimeListening > 0) { - if (!listeningStats.days[s.date]) listeningStats.days[s.date] = 0 - listeningStats.days[s.date] += sessionTimeListening + if (sessionDate && sessionTimeListening > 0) { + if (!listeningStats.days[sessionDate]) listeningStats.days[sessionDate] = 0 + listeningStats.days[sessionDate] += sessionTimeListening - if (s.date === today) { + if (sessionDate === today) { listeningStats.today += sessionTimeListening } } - if (!listeningStats.items[s.libraryItemId]) { - listeningStats.items[s.libraryItemId] = { - id: s.libraryItemId, - timeListening: sessionTimeListening, - mediaMetadata: s.mediaMetadata, - lastUpdate: s.lastUpdate + if (!libraryItemId) { + listeningStats.totalTime += sessionTimeListening + return + } + + if (includeItems) { + if (!listeningStats.items[libraryItemId]) { + listeningStats.items[libraryItemId] = { + id: libraryItemId, + timeListening: sessionTimeListening, + mediaMetadata: sessionMediaMetadata, + lastUpdate: sessionUpdatedAt + } + } else { + listeningStats.items[libraryItemId].timeListening += sessionTimeListening } - } else { - listeningStats.items[s.libraryItemId].timeListening += sessionTimeListening } listeningStats.totalTime += sessionTimeListening }) + Logger.debug( + `[ApiRouter] Listening stats for user "${userId}" aggregated ${listeningSessions.length} sessions in ${Date.now() - startedAt}ms includeItems=${includeItems} includeRecentSessions=${includeRecentSessions}` + ) return listeningStats } } diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 73cb51553..a34750f07 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -14,6 +14,7 @@ const LibraryItemScanner = require('./LibraryItemScanner') const LibraryScan = require('./LibraryScan') const LibraryItemScanData = require('./LibraryItemScanData') const Task = require('../objects/Task') +const { isPostgres, jsonArrayExpand } = require('../utils/sqlDialectHelpers') class LibraryScanner { constructor() { @@ -676,12 +677,16 @@ async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingle // check if it was moved from another folder by comparing the ino to the library files const ino = await fileUtils.getIno(fullPath) if (!ino) return null + const inodeMatchQuery = isPostgres(Database.sequelize) + ? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{ino}' = :inode)` + : `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)` + const existingLibraryItem = await Database.libraryItemModel.findOneExpanded( [ { libraryId: libraryId }, - sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), { + sequelize.where(sequelize.literal(inodeMatchQuery), { [sequelize.Op.gt]: 0 }) ], diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js new file mode 100644 index 000000000..1b6adcd7d --- /dev/null +++ b/server/scripts/migrateSqliteToPostgres.js @@ -0,0 +1,515 @@ +#!/usr/bin/env node + +const sqlite3 = require('sqlite3') +const { Client } = require('pg') + +const SQLITE_PATH = process.env.SQLITE_PATH || '/config/absdatabase.sqlite' +const DATABASE_URL = process.env.DATABASE_URL +const PG_SCHEMA = process.env.PG_SCHEMA || 'public' +const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500) +const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true' +const ALLOW_DESTRUCTIVE_TARGET = String(process.env.ALLOW_DESTRUCTIVE_TARGET || 'false').toLowerCase() === 'true' + +const integerBounds = { + smallint: { min: -32768n, max: 32767n }, + integer: { min: -2147483648n, max: 2147483647n }, + bigint: { min: -9223372036854775808n, max: 9223372036854775807n } +} + +const preferredOrder = [ + 'migrationsMeta', + 'SequelizeMeta', + 'settings', + 'users', + 'apiKeys', + 'sessions', + 'libraries', + 'libraryFolders', + 'authors', + 'series', + 'books', + 'podcasts', + 'podcastEpisodes', + 'libraryItems', + 'bookAuthors', + 'bookSeries', + 'collections', + 'collectionBooks', + 'playlists', + 'playlistMediaItems', + 'mediaProgresses', + 'devices', + 'playbackSessions', + 'feeds', + 'feedEpisodes', + 'mediaItemShares', + 'customMetadataProviders' +] + +function quoteIdent(identifier) { + return `"${String(identifier).replace(/"/g, '""')}"` +} + +function openSqlite(filePath) { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(filePath, sqlite3.OPEN_READONLY, (err) => { + if (err) return reject(err) + resolve(db) + }) + }) +} + +function sqliteAll(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => { + if (err) return reject(err) + resolve(rows) + }) + }) +} + +function sqliteGet(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.get(sql, params, (err, row) => { + if (err) return reject(err) + resolve(row) + }) + }) +} + +async function findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) { + const issues = [] + + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const pgColumns = pgColumnsByTable.get(postgresTable) + if (!pgColumns) continue + + for (const column of pgColumns.values()) { + if (column.data_type !== 'character varying' || !column.character_maximum_length) continue + + const sqliteColumn = column.column_name + const maxLength = Number(column.character_maximum_length) + const quotedTable = quoteIdent(sqliteTable) + const quotedColumn = quoteIdent(sqliteColumn) + + try { + const maxLengthRow = await sqliteGet( + sqliteDb, + `SELECT MAX(LENGTH(${quotedColumn})) AS maxLength FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL` + ) + const actualMaxLength = Number(maxLengthRow?.maxLength || 0) + if (actualMaxLength <= maxLength) continue + + const overCountRow = await sqliteGet( + sqliteDb, + `SELECT COUNT(*) AS count FROM ${quotedTable} WHERE LENGTH(${quotedColumn}) > ?`, + [maxLength] + ) + + issues.push({ + sqliteTable, + sqliteColumn, + postgresTable, + postgresColumn: column.column_name, + maxLength, + actualMaxLength, + overCount: Number(overCountRow?.count || 0) + }) + } catch (error) { + // Ignore columns missing in sqlite source table + } + } + } + + return issues +} + +function parseIntegerValue(value) { + if (typeof value === 'number') { + if (!Number.isSafeInteger(value)) return null + return BigInt(value) + } + + if (typeof value === 'string') { + const trimmed = value.trim() + if (!trimmed || !/^-?\d+$/.test(trimmed)) return null + try { + return BigInt(trimmed) + } catch (error) { + return null + } + } + + return null +} + +function isIntegerCompatible(value, dataType = 'bigint') { + if (value === null || value === undefined) return true + + const parsed = parseIntegerValue(value) + if (parsed === null) return false + + const bounds = integerBounds[dataType] + if (!bounds) return false + return parsed >= bounds.min && parsed <= bounds.max +} + +async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) { + const issues = [] + + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const pgColumns = pgColumnsByTable.get(postgresTable) + if (!pgColumns) continue + + for (const column of pgColumns.values()) { + const dataType = column.data_type + if (dataType !== 'smallint' && dataType !== 'integer' && dataType !== 'bigint') continue + + const sqliteColumn = column.column_name + const quotedTable = quoteIdent(sqliteTable) + const quotedColumn = quoteIdent(sqliteColumn) + + let rows + try { + rows = await sqliteAll(sqliteDb, `SELECT ${quotedColumn} AS value FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`) + } catch (error) { + // Ignore columns missing in sqlite source table + continue + } + + let badCount = 0 + let sampleValue = null + for (const row of rows) { + if (!isIntegerCompatible(row.value, dataType)) { + badCount += 1 + if (sampleValue === null) sampleValue = row.value + } + } + + if (badCount > 0) { + issues.push({ + sqliteTable, + sqliteColumn, + postgresTable, + postgresColumn: column.column_name, + postgresType: dataType, + badCount, + sampleValue + }) + } + } + } + + return issues +} + +function normalizeBoolean(value) { + if (value === null || value === undefined) return null + if (typeof value === 'boolean') return value + if (typeof value === 'number') return value !== 0 + if (typeof value === 'string') { + const v = value.trim().toLowerCase() + return v === 'true' || v === '1' || v === 't' + } + return !!value +} + +function normalizeJson(value) { + if (value === null || value === undefined || value === '') return null + if (typeof value === 'object') return JSON.stringify(value) + + const textValue = String(value) + + try { + const parsed = JSON.parse(textValue) + if (typeof parsed === 'string') { + try { + return JSON.stringify(JSON.parse(parsed)) + } catch (error) { + return JSON.stringify(parsed) + } + } + return JSON.stringify(parsed) + } catch (error) { + // Keep non-JSON payloads as JSON string values so inserts remain valid JSON. + return JSON.stringify(textValue) + } +} + +function convertValue(value, pgColumn) { + if (!pgColumn) return value + const dataType = pgColumn.data_type + const udtName = pgColumn.udt_name + + if (dataType === 'boolean') { + return normalizeBoolean(value) + } + + if (dataType === 'json' || dataType === 'jsonb' || udtName === 'json' || udtName === 'jsonb') { + return normalizeJson(value) + } + + if ((dataType === 'smallint' || dataType === 'integer' || dataType === 'bigint') && value !== null && value !== undefined) { + if (!isIntegerCompatible(value, dataType)) return value + if (dataType === 'bigint' && typeof value === 'string') { + return value.trim() + } + if (typeof value === 'number') return value + return Number(value) + } + + return value +} + +function getOverlongColumns(row, insertColumns) { + const overlong = [] + + for (const column of insertColumns) { + const maxLength = column.metadata.character_maximum_length + if (!maxLength) continue + + const value = row[column.sqliteColumn] + if (value === null || value === undefined) continue + + const length = String(value).length + if (length > maxLength) { + overlong.push({ + sqliteColumn: column.sqliteColumn, + postgresColumn: column.postgresColumn, + maxLength, + actualLength: length + }) + } + } + + return overlong +} + +async function main() { + if (!DATABASE_URL) { + throw new Error('DATABASE_URL is required') + } + + console.log(`[migrate] sqlite source: ${SQLITE_PATH}`) + console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`) + console.log(`[migrate] dry run: ${DRY_RUN}`) + + const sqliteDb = await openSqlite(SQLITE_PATH) + const pg = new Client({ connectionString: DATABASE_URL }) + await pg.connect() + + try { + const sqliteTablesRows = await sqliteAll( + sqliteDb, + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + ) + const sqliteTables = sqliteTablesRows.map((row) => row.name) + + const pgTablesRows = await pg.query( + `SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`, + [PG_SCHEMA] + ) + + const pgTablesByLowerName = new Map(pgTablesRows.rows.map((row) => [row.table_name.toLowerCase(), row.table_name])) + + const tablesToMigrate = sqliteTables + .map((sqliteTable) => { + const postgresTable = pgTablesByLowerName.get(sqliteTable.toLowerCase()) + if (!postgresTable) return null + return { + sqliteTable, + postgresTable + } + }) + .filter(Boolean) + + tablesToMigrate.sort((a, b) => { + const ai = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === a.sqliteTable.toLowerCase()) + const bi = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === b.sqliteTable.toLowerCase()) + if (ai === -1 && bi === -1) return a.sqliteTable.localeCompare(b.sqliteTable) + if (ai === -1) return 1 + if (bi === -1) return -1 + return ai - bi + }) + + if (!tablesToMigrate.length) { + throw new Error('No overlapping tables found between SQLite and PostgreSQL') + } + + console.log(`[migrate] tables to migrate: ${tablesToMigrate.map((table) => table.sqliteTable).join(', ')}`) + + const pgColumnsByTable = new Map() + for (const { postgresTable } of tablesToMigrate) { + const columnsResult = await pg.query( + `SELECT column_name, data_type, udt_name, character_maximum_length FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, + [PG_SCHEMA, postgresTable] + ) + const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column])) + pgColumnsByTable.set(postgresTable, columnMap) + } + + const overlongVarcharIssues = await findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) + const integerTypeIssues = await findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) + + if (overlongVarcharIssues.length) { + console.error('[migrate] overlong source values detected for varchar columns:') + overlongVarcharIssues.forEach((issue) => { + console.error( + `[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` + + `(max=${issue.maxLength}, actualMax=${issue.actualMaxLength}, overRows=${issue.overCount})` + ) + }) + throw new Error('Migration aborted to prevent truncation/data loss. Widen target column types first.') + } + + if (integerTypeIssues.length) { + console.error('[migrate] non-integer source values detected for integer columns:') + integerTypeIssues.forEach((issue) => { + console.error( + `[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` + + `(type=${issue.postgresType}, badRows=${issue.badCount}, sample=${JSON.stringify(issue.sampleValue)})` + ) + }) + throw new Error('Migration aborted to prevent numeric precision loss. Widen target numeric column types first.') + } + + if (!DRY_RUN) { + if (!ALLOW_DESTRUCTIVE_TARGET) { + throw new Error('Migration writes are destructive. Set ALLOW_DESTRUCTIVE_TARGET=true after confirming the target database can be truncated.') + } + + await pg.query('BEGIN') + await pg.query('SET session_replication_role = replica') + + const truncateList = tablesToMigrate.map(({ postgresTable }) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`).join(', ') + await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`) + console.log('[migrate] truncated target tables') + } + + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(sqliteTable)}`) + const pgColumns = pgColumnsByTable.get(postgresTable) + const insertColumns = rows.length + ? Object.keys(rows[0]) + .map((sqliteColumn) => { + const pgColumn = pgColumns.get(sqliteColumn.toLowerCase()) + if (!pgColumn) return null + return { + sqliteColumn, + postgresColumn: pgColumn.column_name, + metadata: pgColumn + } + }) + .filter(Boolean) + : [] + + if (!rows.length || !insertColumns.length) { + console.log(`[migrate] ${sqliteTable}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`) + continue + } + + if (!DRY_RUN) { + for (let offset = 0; offset < rows.length; offset += BATCH_SIZE) { + const batchRows = rows.slice(offset, offset + BATCH_SIZE) + const valuesSql = [] + const params = [] + let paramIndex = 1 + + for (const row of batchRows) { + const placeholders = [] + for (const column of insertColumns) { + params.push(convertValue(row[column.sqliteColumn], column.metadata)) + placeholders.push(`$${paramIndex++}`) + } + valuesSql.push(`(${placeholders.join(', ')})`) + } + + const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}` + + try { + await pg.query('SAVEPOINT migrate_batch') + await pg.query(insertSql, params) + await pg.query('RELEASE SAVEPOINT migrate_batch') + } catch (error) { + await pg.query('ROLLBACK TO SAVEPOINT migrate_batch') + console.error(`[migrate] batch insert failed for ${sqliteTable} (offset=${offset}, size=${batchRows.length}): ${error.message}`) + + for (let rowIndex = 0; rowIndex < batchRows.length; rowIndex++) { + const row = batchRows[rowIndex] + const singleRowParams = insertColumns.map((column) => convertValue(row[column.sqliteColumn], column.metadata)) + const singleRowInsertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES (${singleRowParams.map((_, index) => `$${index + 1}`).join(', ')})` + + try { + await pg.query('SAVEPOINT migrate_row') + await pg.query(singleRowInsertSql, singleRowParams) + await pg.query('RELEASE SAVEPOINT migrate_row') + } catch (rowError) { + await pg.query('ROLLBACK TO SAVEPOINT migrate_row') + const overlongColumns = getOverlongColumns(row, insertColumns) + if (overlongColumns.length) { + overlongColumns.forEach((column) => { + console.error( + `[migrate] overlong value in ${sqliteTable}.${column.sqliteColumn} -> ${postgresTable}.${column.postgresColumn} ` + + `(length=${column.actualLength}, max=${column.maxLength})` + ) + }) + } + + throw rowError + } + } + } + } + } + + console.log(`[migrate] ${sqliteTable}: ${rows.length} rows`) + } + + if (!DRY_RUN) { + await pg.query('SET session_replication_role = DEFAULT') + await pg.query('COMMIT') + console.log('[migrate] migration transaction committed') + } + + const parity = [] + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(sqliteTable)}`) + const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`) + parity.push({ + table: sqliteTable, + sqliteCount: Number(sqliteCountRow.count || 0), + postgresCount: Number(pgCountResult.rows[0].count || 0) + }) + } + + const mismatches = parity.filter((row) => row.sqliteCount !== row.postgresCount) + if (mismatches.length) { + console.error('[migrate] row-count mismatches detected:') + mismatches.forEach((row) => { + console.error(`[migrate] ${row.table}: sqlite=${row.sqliteCount} postgres=${row.postgresCount}`) + }) + process.exitCode = 2 + } else { + console.log('[migrate] parity check passed for all migrated tables') + } + } finally { + sqliteDb.close() + await pg.end() + } +} + +if (require.main === module) { + main().catch((error) => { + console.error('[migrate] failed:', error) + process.exit(1) + }) +} + +module.exports = { + normalizeJson, + isIntegerCompatible, + convertValue, + findOverlongVarcharValues, + findIntegerTypeIssues, + quoteIdent +} diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index 7312b9d5d..501f92dc0 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -5,10 +5,22 @@ const libraryItemsBookFilters = require('./libraryItemsBookFilters') const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters') const { createNewSortInstance } = require('../../libs/fastSort') const { profile } = require('../../utils/profiler') +const { booleanLiteral, jsonArrayContainsAny } = require('../sqlDialectHelpers') const naturalSort = createNewSortInstance({ comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare }) +async function withShelfFallback(scope, fallbackValue, action) { + try { + return await action() + } catch (error) { + // Only postgres shelves fall back to empty results - on sqlite keep upstream behavior of surfacing the error + if (!Database.isPostgresDialect()) throw error + Logger.error(`[LibraryFilters] Failed to load ${scope}`, error) + return fallbackValue + } +} + module.exports = { decode(text) { return Buffer.from(decodeURIComponent(text), 'base64').toString() @@ -49,22 +61,24 @@ module.exports = { * @returns {Promise<{ items:import('../../models/LibraryItem')[], count:number }>} */ async getMediaItemsInProgress(library, user, include, limit) { - if (library.isBook) { - const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'in-progress', 'progress', true, false, include, limit, 0, true) - return { - items: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - if (li.rssFeed) { - oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() - } - if (li.mediaItemShare) { - oldLibraryItem.mediaItemShare = li.mediaItemShare - } - return oldLibraryItem - }), - count + return withShelfFallback(`in-progress shelf for library "${library.id}"`, { items: [], count: 0 }, async () => { + if (library.isBook) { + const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'in-progress', 'progress', true, false, include, limit, 0, true) + return { + items: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + if (li.rssFeed) { + oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() + } + if (li.mediaItemShare) { + oldLibraryItem.mediaItemShare = li.mediaItemShare + } + return oldLibraryItem + }), + count + } } - } else { + const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'in-progress', 'progress', true, limit, 0, true) return { count, @@ -74,7 +88,7 @@ module.exports = { return oldLibraryItem }) } - } + }) }, /** @@ -86,25 +100,27 @@ module.exports = { * @returns {object} { libraryItems:LibraryItem[], count:number } */ async getLibraryItemsMostRecentlyAdded(library, user, include, limit) { - if (library.isBook) { - const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, false, include, limit, 0) - return { - libraryItems: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - if (li.rssFeed) { - oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() - } - if (li.size && !oldLibraryItem.media.size) { - oldLibraryItem.media.size = li.size - } - if (li.mediaItemShare) { - oldLibraryItem.mediaItemShare = li.mediaItemShare - } - return oldLibraryItem - }), - count + return withShelfFallback(`recent shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => { + if (library.isBook) { + const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, false, include, limit, 0) + return { + libraryItems: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + if (li.rssFeed) { + oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() + } + if (li.size && !oldLibraryItem.media.size) { + oldLibraryItem.media.size = li.size + } + if (li.mediaItemShare) { + oldLibraryItem.mediaItemShare = li.mediaItemShare + } + return oldLibraryItem + }), + count + } } - } else { + const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, include, limit, 0) return { libraryItems: libraryItems.map((li) => { @@ -122,7 +138,7 @@ module.exports = { }), count } - } + }) }, /** @@ -134,23 +150,25 @@ module.exports = { * @returns {object} { libraryItems:LibraryItem[], count:number } */ async getLibraryItemsContinueSeries(library, user, include, limit) { - const { libraryItems, count } = await libraryItemsBookFilters.getContinueSeriesLibraryItems(library, user, include, limit, 0) - return { - libraryItems: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - if (li.rssFeed) { - oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() - } - if (li.series) { - oldLibraryItem.media.metadata.series = li.series - } - if (li.mediaItemShare) { - oldLibraryItem.mediaItemShare = li.mediaItemShare - } - return oldLibraryItem - }), - count - } + return withShelfFallback(`continue-series shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => { + const { libraryItems, count } = await libraryItemsBookFilters.getContinueSeriesLibraryItems(library, user, include, limit, 0) + return { + libraryItems: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + if (li.rssFeed) { + oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() + } + if (li.series) { + oldLibraryItem.media.metadata.series = li.series + } + if (li.mediaItemShare) { + oldLibraryItem.mediaItemShare = li.mediaItemShare + } + return oldLibraryItem + }), + count + } + }) }, /** @@ -163,22 +181,24 @@ module.exports = { * @returns {Promise<{ items:oldLibraryItem[], count:number }>} */ async getMediaFinished(library, user, include, limit) { - if (library.isBook) { - const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'finished', 'progress', true, false, include, limit, 0) - return { - items: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - if (li.rssFeed) { - oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() - } - if (li.mediaItemShare) { - oldLibraryItem.mediaItemShare = li.mediaItemShare - } - return oldLibraryItem - }), - count + return withShelfFallback(`finished shelf for library "${library.id}"`, { items: [], count: 0 }, async () => { + if (library.isBook) { + const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'finished', 'progress', true, false, include, limit, 0) + return { + items: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + if (li.rssFeed) { + oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() + } + if (li.mediaItemShare) { + oldLibraryItem.mediaItemShare = li.mediaItemShare + } + return oldLibraryItem + }), + count + } } - } else { + const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'finished', 'progress', true, limit, 0) return { count, @@ -188,7 +208,7 @@ module.exports = { return oldLibraryItem }) } - } + }) }, /** @@ -202,23 +222,24 @@ module.exports = { async getSeriesMostRecentlyAdded(library, user, include, limit) { if (!library.isBook) return { series: [], count: 0 } - const seriesIncludes = [] - if (include.includes('rssfeed')) { - seriesIncludes.push({ - model: Database.feedModel - }) - } - - const userPermissionBookWhere = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user) - - const seriesWhere = [ - { - libraryId: library.id, - createdAt: { - [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago - } + return withShelfFallback(`recent-series shelf for library "${library.id}"`, { series: [], count: 0 }, async () => { + const seriesIncludes = [] + if (include.includes('rssfeed')) { + seriesIncludes.push({ + model: Database.feedModel + }) } - ] + + const userPermissionBookWhere = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user) + + const seriesWhere = [ + { + libraryId: library.id, + createdAt: { + [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago + } + } + ] // Handle library setting to hide single book series // TODO: Merge with existing query @@ -235,13 +256,13 @@ module.exports = { if (userPermissionBookWhere.bookWhere.length) { let attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id' if (!user.canAccessExplicitContent) { - attrQuery += ' AND b.explicit = 0' + attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}` } if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (user.permissions.selectedTagsNotAccessible) { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0` } else { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0` } } seriesWhere.push( @@ -251,29 +272,29 @@ module.exports = { ) } - const { rows: series, count } = await Database.seriesModel.findAndCountAll({ - where: seriesWhere, - limit, - offset: 0, - distinct: true, - subQuery: false, - replacements: userPermissionBookWhere.replacements, - include: [ - { - model: Database.bookSeriesModel, - include: { - model: Database.bookModel, - where: userPermissionBookWhere.bookWhere, + const { rows: series, count } = await Database.seriesModel.findAndCountAll({ + where: seriesWhere, + limit, + offset: 0, + distinct: true, + subQuery: false, + replacements: userPermissionBookWhere.replacements, + include: [ + { + model: Database.bookSeriesModel, include: { - model: Database.libraryItemModel - } + model: Database.bookModel, + where: userPermissionBookWhere.bookWhere, + include: { + model: Database.libraryItemModel + } + }, + separate: true }, - separate: true - }, - ...seriesIncludes - ], - order: [['createdAt', 'DESC']] - }) + ...seriesIncludes + ], + order: [['createdAt', 'DESC']] + }) const allOldSeries = [] for (const s of series) { @@ -311,10 +332,11 @@ module.exports = { allOldSeries.push(oldSeries) } - return { - series: allOldSeries, - count - } + return { + series: allOldSeries, + count + } + }) }, /** @@ -329,37 +351,39 @@ module.exports = { async getNewestAuthors(library, user, limit) { if (library.mediaType !== 'book') return { authors: [], count: 0 } - const { bookWhere, replacements } = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user) + return withShelfFallback(`newest-authors shelf for library "${library.id}"`, { authors: [], count: 0 }, async () => { + const { bookWhere, replacements } = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user) - const { rows: authors, count } = await Database.authorModel.findAndCountAll({ - where: { - libraryId: library.id, - createdAt: { - [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago - } - }, - replacements, - include: { - model: Database.bookModel, - attributes: ['id', 'tags', 'explicit'], - where: bookWhere, - required: true, // Must belong to a book - through: { - attributes: [] - } - }, - limit, - distinct: true, - order: [['createdAt', 'DESC']] + const { rows: authors, count } = await Database.authorModel.findAndCountAll({ + where: { + libraryId: library.id, + createdAt: { + [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago + } + }, + replacements, + include: { + model: Database.bookModel, + attributes: ['id', 'tags', 'explicit'], + where: bookWhere, + required: true, // Must belong to a book + through: { + attributes: [] + } + }, + limit, + distinct: true, + order: [['createdAt', 'DESC']] + }) + + return { + authors: authors.map((au) => { + const numBooks = au.books.length || 0 + return au.toOldJSONExpanded(numBooks) + }), + count + } }) - - return { - authors: authors.map((au) => { - const numBooks = au.books.length || 0 - return au.toOldJSONExpanded(numBooks) - }), - count - } }, /** @@ -373,20 +397,22 @@ module.exports = { async getLibraryItemsToDiscover(library, user, include, limit) { if (library.mediaType !== 'book') return { libraryItems: [], count: 0 } - const { libraryItems, count } = await libraryItemsBookFilters.getDiscoverLibraryItems(library.id, user, include, limit) - return { - libraryItems: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - if (li.rssFeed) { - oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() - } - if (li.mediaItemShare) { - oldLibraryItem.mediaItemShare = li.mediaItemShare - } - return oldLibraryItem - }), - count - } + return withShelfFallback(`discover shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => { + const { libraryItems, count } = await libraryItemsBookFilters.getDiscoverLibraryItems(library.id, user, include, limit) + return { + libraryItems: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + if (li.rssFeed) { + oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() + } + if (li.mediaItemShare) { + oldLibraryItem.mediaItemShare = li.mediaItemShare + } + return oldLibraryItem + }), + count + } + }) }, /** @@ -399,15 +425,17 @@ module.exports = { async getNewestPodcastEpisodes(library, user, limit) { if (library.mediaType !== 'podcast') return { libraryItems: [], count: 0 } - const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'recent', null, 'createdAt', true, limit, 0) - return { - count, - libraryItems: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - oldLibraryItem.recentEpisode = li.recentEpisode - return oldLibraryItem - }) - } + return withShelfFallback(`newest-podcast-episodes shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => { + const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'recent', null, 'createdAt', true, limit, 0) + return { + count, + libraryItems: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + oldLibraryItem.recentEpisode = li.recentEpisode + return oldLibraryItem + }) + } + }) }, /** diff --git a/server/utils/queries/libraryItemFilters.js b/server/utils/queries/libraryItemFilters.js index 7f95d0ecc..15c3dedb7 100644 --- a/server/utils/queries/libraryItemFilters.js +++ b/server/utils/queries/libraryItemFilters.js @@ -2,6 +2,7 @@ const Sequelize = require('sequelize') const Database = require('../../Database') const libraryItemsBookFilters = require('./libraryItemsBookFilters') const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters') +const { jsonArrayContainsAny } = require('../sqlDialectHelpers') module.exports = { /** @@ -12,7 +13,7 @@ module.exports = { async getAllLibraryItemsWithTags(tags) { const libraryItems = [] const booksWithTag = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -46,7 +47,7 @@ module.exports = { libraryItems.push(libraryItem) } const podcastsWithTag = await Database.podcastModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -77,7 +78,7 @@ module.exports = { async getAllLibraryItemsWithGenres(genres) { const libraryItems = [] const booksWithGenre = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -107,7 +108,7 @@ module.exports = { libraryItems.push(libraryItem) } const podcastsWithGenre = await Database.podcastModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -138,7 +139,7 @@ module.exports = { async getAllLibraryItemsWithNarrators(narrators) { const libraryItems = [] const booksWithGenre = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(narrators) WHERE json_valid(narrators) AND json_each.value IN (:narrators))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('narrators', 'narrators', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index fbe0c4f0d..18f0918ad 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -6,6 +6,7 @@ const authorFilters = require('./authorFilters') const ShareManager = require('../../managers/ShareManager') const { profile } = require('../profiler') const stringifySequelizeQuery = require('../stringifySequelizeQuery') +const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand, safeTextToDoubleExpression, safeTextToIntegerExpression } = require('../sqlDialectHelpers') const countCache = new Map() module.exports = { @@ -27,10 +28,10 @@ module.exports = { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { replacements['userTagsSelected'] = user.permissions.itemTagsSelected if (user.permissions.selectedTagsNotAccessible) { - bookWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0)) + bookWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0)) } else { bookWhere.push( - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), { + Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) ) @@ -189,7 +190,7 @@ module.exports = { } else if (group === 'explicit') { mediaWhere['explicit'] = true } else if (['genres', 'tags', 'narrators'].includes(group)) { - mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), { + mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) replacements.filterValue = value @@ -236,7 +237,7 @@ module.exports = { } else if (group === 'publishedDecades') { const startYear = parseInt(value) const endYear = parseInt(value, 10) + 9 - mediaWhere = Sequelize.where(Sequelize.literal('CAST(publishedYear AS INTEGER)'), { + mediaWhere = Sequelize.where(Sequelize.literal(safeTextToIntegerExpression('publishedYear', Database.sequelize)), { [Sequelize.Op.between]: [startYear, endYear] }) } @@ -256,9 +257,9 @@ module.exports = { const getTitleOrder = () => { if (global.ServerSettings.sortingIgnorePrefix) { - return [Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir] + return [Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir] } else { - return [Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir] + return [Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir] } } @@ -273,21 +274,22 @@ module.exports = { } else if (sortBy === 'media.duration') { return [['duration', dir]] } else if (sortBy === 'media.metadata.publishedYear') { - return [[Sequelize.literal(`CAST(\`book\`.\`publishedYear\` AS INTEGER)`), dir]] + return [[Sequelize.literal(safeTextToIntegerExpression('book.publishedYear', Database.sequelize)), dir]] } else if (sortBy === 'media.metadata.authorNameLF') { // Sort by author name last first, secondary sort by title - return [[Sequelize.literal('`libraryItem`.`authorNamesLastFirst` COLLATE NOCASE'), dir], getTitleOrder()] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesLastFirst', Database.sequelize)), dir], getTitleOrder()] } else if (sortBy === 'media.metadata.authorName') { // Sort by author name first last, secondary sort by title - return [[Sequelize.literal('`libraryItem`.`authorNamesFirstLast` COLLATE NOCASE'), dir], getTitleOrder()] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesFirstLast', Database.sequelize)), dir], getTitleOrder()] } else if (sortBy === 'media.metadata.title') { if (collapseseries) { - return [[Sequelize.literal('display_title COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('display_title', Database.sequelize)), dir]] } return [getTitleOrder()] } else if (sortBy === 'sequence') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' - return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]] + const sequenceColumn = Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`' + return [[Sequelize.literal(`${safeTextToDoubleExpression(sequenceColumn, Database.sequelize)} ${nullDir}`)]] } else if (sortBy === 'progress') { return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]] } else if (sortBy === 'progress.createdAt') { @@ -326,7 +328,9 @@ module.exports = { required: true } ], - order: [Sequelize.literal('CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')] + order: [ + Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"books->bookSeries"."sequence"' : '`books.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) + ] }) const bookSeriesToInclude = [] const booksToInclude = [] @@ -358,26 +362,32 @@ module.exports = { async findAndCountAll(findOptions, limit, offset, useCountCache) { const model = Database.bookModel - if (useCountCache) { - const countCacheKey = stringifySequelizeQuery(findOptions) - Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`) - if (!countCache.has(countCacheKey)) { - const count = await model.count(findOptions) - countCache.set(countCacheKey, count) - } - - findOptions.limit = limit || null - findOptions.offset = offset - - const rows = await model.findAll(findOptions) - - return { rows, count: countCache.get(countCacheKey) } - } - findOptions.limit = limit || null findOptions.offset = offset - return await model.findAndCountAll(findOptions) + try { + if (useCountCache) { + const countCacheKey = stringifySequelizeQuery(findOptions) + Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`) + if (!countCache.has(countCacheKey)) { + const count = await model.count(findOptions) + countCache.set(countCacheKey, count) + } + + const rows = await model.findAll(findOptions) + return { rows, count: countCache.get(countCacheKey) } + } + + return await model.findAndCountAll(findOptions) + } catch (error) { + Logger.error(`[LibraryItemsBookFilters] findAndCountAll failed: ${error.message}`) + try { + Logger.error(`[LibraryItemsBookFilters] findAndCountAll query: ${stringifySequelizeQuery(findOptions)}`) + } catch (stringifyError) { + Logger.error(`[LibraryItemsBookFilters] failed to stringify query: ${stringifyError.message}`) + } + throw error + } }, /** @@ -455,12 +465,28 @@ module.exports = { }) } else if (filterGroup === 'ebooks' && filterValue === 'supplementary') { // TODO: Temp workaround for filtering supplementary ebook - libraryItemWhere['libraryFiles'] = { - [Sequelize.Op.substring]: `"isSupplementary":true` + if (Database.sequelize.getDialect() === 'postgres') { + libraryItemWhere[Sequelize.Op.and] = [ + Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), { + [Sequelize.Op.like]: '%"isSupplementary":true%' + }) + ] + } else { + libraryItemWhere['libraryFiles'] = { + [Sequelize.Op.substring]: `"isSupplementary":true` + } } } else if (filterGroup === 'ebooks' && filterValue === 'no-supplementary') { - libraryItemWhere['libraryFiles'] = { - [Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`) + if (Database.sequelize.getDialect() === 'postgres') { + libraryItemWhere[Sequelize.Op.and] = [ + Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), { + [Sequelize.Op.notLike]: '%"isSupplementary":true%' + }) + ] + } else { + libraryItemWhere['libraryFiles'] = { + [Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`) + } } } else if (filterGroup === 'missing' && filterValue === 'authors') { authorInclude = { @@ -502,7 +528,9 @@ module.exports = { }) if (sortBy !== 'sequence') { // Secondary sort by sequence - sortOrder.push([Sequelize.literal('CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]) + sortOrder.push([ + Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) + ]) } } else if (filterGroup === 'issues') { libraryItemWhere[Sequelize.Op.or] = [ @@ -594,10 +622,32 @@ module.exports = { // When collapsing series and sorting by title then use the series name instead of the book title // for this set an attribute "display_title" to use in sorting + const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.title' : '`libraryItem`.`title`' + const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.titleIgnorePrefix' : '`libraryItem`.`titleIgnorePrefix`' + const fallbackFn = coalesceFunctionName(Database.sequelize) + const includedBookSeriesIds = bookSeriesToInclude.map((v) => Database.sequelize.escape(v.id)).join(', ') + const collapseSeriesSubqueryByName = includedBookSeriesIds + ? `(SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))` + : 'NULL' + const collapseSeriesSubqueryByNameIgnorePrefix = includedBookSeriesIds + ? `(SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))` + : 'NULL' + + Logger.debug( + `[LibraryItemsBookFilters] collapse-series computed includeIds=${bookSeriesToInclude.length} excludeBooks=${booksToExclude.length} ` + + `filterGroup=${filterGroup || 'none'} filterValue=${filterValue || 'none'} sortBy=${sortBy}` + ) + if (!includedBookSeriesIds) { + Logger.debug( + `[LibraryItemsBookFilters] collapse-series produced no include IDs; using library item title fallback ` + + `(libraryId=${libraryId}, filterGroup=${filterGroup || 'none'}, filterValue=${filterValue || 'none'}, sortBy=${sortBy})` + ) + } + if (global.ServerSettings.sortingIgnorePrefix) { - bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`titleIgnorePrefix\`)`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByNameIgnorePrefix}, ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) } else { - bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`title\`)`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByName}, ${fallbackLibraryItemTitle})`), 'display_title']) } } @@ -716,13 +766,14 @@ module.exports = { bookWhere.push(...userPermissionBookWhere.bookWhere) let includeAttributes = [[Sequelize.literal('(SELECT max(mp.updatedAt) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id)'), 'recent_progress']] - let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = 0 OR mp.isFinished IS NULL)` + let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = ${booleanLiteral(false, Database.sequelize)} OR mp.isFinished IS NULL)` if (library.settings.onlyShowLaterBooksInContinueSeries) { - const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as FLOAT) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = 1 AND mp.userId = :userId AND bs.seriesId = series.id)` + const safeSequenceExpr = safeTextToDoubleExpression('bs.sequence', Database.sequelize) + const maxSequenceQuery = `(SELECT max(${safeSequenceExpr}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)` includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence']) - booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as FLOAT) > ${maxSequenceQuery}` + booksNotFinishedQuery = booksNotFinishedQuery + ` AND ${safeSequenceExpr} > ${maxSequenceQuery}` } const { rows: series, count } = await Database.seriesModel.findAndCountAll({ @@ -735,7 +786,7 @@ module.exports = { }, // TODO: Simplify queries // Has at least 1 book finished - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = 1)`), { + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)})`), { [Sequelize.Op.gte]: 1 }), // Has at least 1 book not finished (that has a sequence number higher than the highest already read, if library config is toggled) @@ -743,7 +794,7 @@ module.exports = { [Sequelize.Op.gte]: 1 }), // Has no books in progress - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = 0 AND mp.currentTime > 0)`), 0) + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)} AND mp.currentTime > 0)`), 0) ], attributes: { include: includeAttributes @@ -757,10 +808,10 @@ module.exports = { attributes: ['bookId', 'sequence'], separate: true, subQuery: false, - order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], + order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]], where: { '$book.mediaProgresses.isFinished$': { - [Sequelize.Op.or]: [null, 0] + [Sequelize.Op.or]: [null, false] } }, include: { @@ -854,7 +905,7 @@ module.exports = { { libraryId }, - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = 1 OR mp.currentTime > 0))`), 0) + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0))`), 0) ], replacements: { userId: user.id, @@ -870,7 +921,7 @@ module.exports = { model: Database.bookModel, where: userPermissionBookWhere.bookWhere }, - order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], + order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]], limit: 1 }, subQuery: false, @@ -891,7 +942,7 @@ module.exports = { const discoverWhere = [ { '$mediaProgresses.isFinished$': { - [Sequelize.Op.or]: [null, 0] + [Sequelize.Op.or]: [null, false] }, '$mediaProgresses.currentTime$': { [Sequelize.Op.or]: [null, 0] @@ -1174,7 +1225,7 @@ module.exports = { // Search narrators const narratorMatches = [] - const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, { + const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, ${jsonArrayExpand('b.narrators', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1191,7 +1242,7 @@ module.exports = { // Search tags const tagMatches = [] - const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.tags', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1208,7 +1259,7 @@ module.exports = { // Search genres const genreMatches = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1288,7 +1339,7 @@ module.exports = { */ async getGenresWithCount(libraryId) { const genres = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { replacements: { libraryId }, diff --git a/server/utils/queries/libraryItemsPodcastFilters.js b/server/utils/queries/libraryItemsPodcastFilters.js index 8bb5dc110..d89a4bd73 100644 --- a/server/utils/queries/libraryItemsPodcastFilters.js +++ b/server/utils/queries/libraryItemsPodcastFilters.js @@ -3,6 +3,7 @@ const Database = require('../../Database') const Logger = require('../../Logger') const { profile } = require('../../utils/profiler') const stringifySequelizeQuery = require('../stringifySequelizeQuery') +const { jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression, jsonArrayExpand, jsonPathNumber } = require('../sqlDialectHelpers') const countCache = new Map() @@ -24,10 +25,10 @@ module.exports = { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { replacements['userTagsSelected'] = user.permissions.itemTagsSelected if (user.permissions.selectedTagsNotAccessible) { - podcastWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0)) + podcastWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0)) } else { podcastWhere.push( - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), { + Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) ) @@ -53,7 +54,7 @@ module.exports = { const replacements = {} if (['genres', 'tags'].includes(group)) { - mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), { + mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) replacements.filterValue = value @@ -87,12 +88,12 @@ module.exports = { return [[Sequelize.literal('libraryItem.mtime'), dir]] } else if (sortBy === 'media.metadata.author') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' - return [[Sequelize.literal(`\`podcast\`.\`author\` COLLATE NOCASE ${nullDir}`)]] + return [[Sequelize.literal(`${noCaseSortExpression('podcast.author', Database.sequelize)} ${nullDir}`)]] } else if (sortBy === 'media.metadata.title') { if (global.ServerSettings.sortingIgnorePrefix) { - return [[Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]] } else { - return [[Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]] } } else if (sortBy === 'media.numTracks') { return [['numEpisodes', dir]] @@ -455,7 +456,7 @@ module.exports = { // Search tags const tagMatches = [] - const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.tags', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -472,7 +473,7 @@ module.exports = { // Search genres const genreMatches = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -563,12 +564,12 @@ module.exports = { * @returns {Promise<{ totalSize:number, totalDuration:number, numAudioFiles:number, totalItems:number}>} */ async getPodcastLibraryStats(libraryId) { - const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = "podcast" AND li.libraryId = :libraryId;`, { + const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = 'podcast' AND li.libraryId = :libraryId;`, { replacements: { libraryId } }) - const [statResults] = await Database.sequelize.query(`SELECT SUM(json_extract(pe.audioFile, '$.duration')) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, { + const [statResults] = await Database.sequelize.query(`SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, { replacements: { libraryId } @@ -588,7 +589,7 @@ module.exports = { */ async getGenresWithCount(libraryId) { const genres = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { replacements: { libraryId }, @@ -611,7 +612,7 @@ module.exports = { */ async getLongestPodcasts(libraryId, limit) { const podcasts = await Database.podcastModel.findAll({ - attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(json_extract(pe.audioFile, '$.duration')) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']], + attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']], include: { model: Database.libraryItemModel, attributes: ['id', 'libraryId'], diff --git a/server/utils/queries/seriesFilters.js b/server/utils/queries/seriesFilters.js index ed71e5b3f..3aa208552 100644 --- a/server/utils/queries/seriesFilters.js +++ b/server/utils/queries/seriesFilters.js @@ -2,6 +2,7 @@ const Sequelize = require('sequelize') const Logger = require('../../Logger') const Database = require('../../Database') const libraryItemsBookFilters = require('./libraryItemsBookFilters') +const { booleanLiteral, jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression } = require('../sqlDialectHelpers') module.exports = { decode(text) { @@ -60,7 +61,7 @@ module.exports = { // TODO: Simplify and break-out let attrQuery = null if (['genres', 'tags', 'narrators'].includes(filterGroup)) { - attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (SELECT count(*) FROM json_each(b.${filterGroup}) WHERE json_valid(b.${filterGroup}) AND json_each.value = :filterValue) > 0` + attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ${jsonArrayContainsValue(`b.${filterGroup}`, 'filterValue', Database.sequelize)} > 0` userPermissionBookWhere.replacements.filterValue = filterValue } else if (filterGroup === 'authors') { attrQuery = 'SELECT count(*) FROM books b, bookSeries bs, bookAuthors ba WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ba.bookId = b.id AND ba.authorId = :filterValue' @@ -73,18 +74,18 @@ module.exports = { userPermissionBookWhere.replacements.filterValue = filterValue } else if (filterGroup === 'progress') { if (filterValue === 'not-finished') { - attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)' + attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})` userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'finished') { - const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)' + const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})` seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0)) userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'not-started') { - const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = 1 OR mp.currentTime > 0)' + const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0)` seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0)) userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'in-progress') { - attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = 0' + attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)}` userPermissionBookWhere.replacements.userId = user.id } } @@ -95,13 +96,13 @@ module.exports = { if (!attrQuery) attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id' if (!user.canAccessExplicitContent) { - attrQuery += ' AND b.explicit = 0' + attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}` } if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (user.permissions.selectedTagsNotAccessible) { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0` } else { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0` } } } @@ -128,9 +129,9 @@ module.exports = { order.push(['createdAt', dir]) } else if (sortBy === 'name') { if (global.ServerSettings.sortingIgnorePrefix) { - order.push([Sequelize.literal('nameIgnorePrefix COLLATE NOCASE'), dir]) + order.push([Sequelize.literal(noCaseSortExpression('nameIgnorePrefix', Database.sequelize)), dir]) } else { - order.push([Sequelize.literal('`series`.`name` COLLATE NOCASE'), dir]) + order.push([Sequelize.literal(noCaseSortExpression('series.name', Database.sequelize)), dir]) } } else if (sortBy === 'totalDuration') { seriesAttributes.include.push([Sequelize.literal('(SELECT SUM(b.duration) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND b.id = bs.bookId)'), 'totalDuration']) diff --git a/server/utils/sqlDialectHelpers.js b/server/utils/sqlDialectHelpers.js new file mode 100644 index 000000000..7f6f93118 --- /dev/null +++ b/server/utils/sqlDialectHelpers.js @@ -0,0 +1,91 @@ +function getDialect(sequelize) { + if (!sequelize) return 'sqlite' + if (typeof sequelize.getDialect === 'function') return sequelize.getDialect() + return sequelize.dialect?.name || 'sqlite' +} + +function isPostgres(sequelize) { + return getDialect(sequelize) === 'postgres' +} + +function booleanLiteral(value, sequelize) { + if (isPostgres(sequelize)) return value ? 'TRUE' : 'FALSE' + return value ? '1' : '0' +} + +function noCaseSortExpression(columnExpression, sequelize) { + if (isPostgres(sequelize)) return `LOWER(${columnExpression})` + return `${columnExpression} COLLATE NOCASE` +} + +function coalesceFunctionName(sequelize) { + return isPostgres(sequelize) ? 'COALESCE' : 'IFNULL' +} + +function jsonArrayContainsAny(columnExpression, bindName, sequelize) { + if (isPostgres(sequelize)) { + return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:${bindName}))` + } + return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value IN (:${bindName}))` +} + +function jsonArrayContainsValue(columnExpression, bindName, sequelize) { + if (isPostgres(sequelize)) { + return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :${bindName})` + } + return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value = :${bindName})` +} + +function jsonArrayExpand(columnExpression, sequelize, options = {}) { + const alias = options.alias || 'json_each' + const textValues = options.textValues !== false + if (isPostgres(sequelize)) { + const fn = textValues ? 'jsonb_array_elements_text' : 'jsonb_array_elements' + return `${fn}(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS ${alias}(value)` + } + return `json_each(${columnExpression})` +} + +function jsonPathText(columnExpression, pathSegments, sequelize) { + const path = Array.isArray(pathSegments) ? pathSegments : [pathSegments] + if (isPostgres(sequelize)) { + return `${columnExpression}::jsonb #>> '{${path.join(',')}}'` + } + return `json_extract(${columnExpression}, '$.${path.join('.')}')` +} + +function jsonPathNumber(columnExpression, pathSegments, sequelize) { + if (isPostgres(sequelize)) { + return `NULLIF(${jsonPathText(columnExpression, pathSegments, sequelize)}, '')::double precision` + } + return `json_extract(${columnExpression}, '$.${[].concat(pathSegments).join('.')}')` +} + +function safeTextToDoubleExpression(columnExpression, sequelize) { + if (isPostgres(sequelize)) { + return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(${columnExpression})::double precision ELSE NULL END` + } + return `CAST(${columnExpression} AS FLOAT)` +} + +function safeTextToIntegerExpression(columnExpression, sequelize) { + if (isPostgres(sequelize)) { + return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?\\d+$' THEN BTRIM(${columnExpression})::integer ELSE NULL END` + } + return `CAST(${columnExpression} AS INTEGER)` +} + +module.exports = { + getDialect, + isPostgres, + booleanLiteral, + noCaseSortExpression, + coalesceFunctionName, + jsonArrayContainsAny, + jsonArrayContainsValue, + jsonArrayExpand, + jsonPathText, + jsonPathNumber, + safeTextToDoubleExpression, + safeTextToIntegerExpression +} diff --git a/test/server/Database.test.js b/test/server/Database.test.js new file mode 100644 index 000000000..047d42a31 --- /dev/null +++ b/test/server/Database.test.js @@ -0,0 +1,160 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const fs = require('../../server/libs/fsExtra') +const Database = require('../../server/Database') + +describe('Database', () => { + let originalDialect + let originalSequelize + let originalEnv + + beforeEach(() => { + originalDialect = Database.dialect + originalSequelize = Database.sequelize + originalEnv = { + DB_DIALECT: process.env.DB_DIALECT, + DATABASE_URL: process.env.DATABASE_URL + } + }) + + afterEach(() => { + Database.dialect = originalDialect + Database.sequelize = originalSequelize + + if (originalEnv.DB_DIALECT === undefined) delete process.env.DB_DIALECT + else process.env.DB_DIALECT = originalEnv.DB_DIALECT + + if (originalEnv.DATABASE_URL === undefined) delete process.env.DATABASE_URL + else process.env.DATABASE_URL = originalEnv.DATABASE_URL + + sinon.restore() + }) + + describe('getConfiguredDialect', () => { + it('should default to sqlite when no env variables are set', () => { + delete process.env.DB_DIALECT + delete process.env.DATABASE_URL + + expect(Database.getConfiguredDialect()).to.equal('sqlite') + }) + + it('should use explicit DB_DIALECT value when valid', () => { + process.env.DB_DIALECT = 'postgres' + process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite' + + expect(Database.getConfiguredDialect()).to.equal('postgres') + }) + + it('should infer postgres dialect from DATABASE_URL', () => { + delete process.env.DB_DIALECT + process.env.DATABASE_URL = 'postgresql://user:pass@localhost:5432/abs' + + expect(Database.getConfiguredDialect()).to.equal('postgres') + }) + + it('should fallback to sqlite for unsupported DB_DIALECT', () => { + process.env.DB_DIALECT = 'mysql' + process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite' + + expect(Database.getConfiguredDialect()).to.equal('sqlite') + }) + }) + + describe('checkHasDb', () => { + it('should not check sqlite file existence in postgres mode', async () => { + Database.dialect = 'postgres' + const pathExistsStub = sinon.stub(fs, 'pathExists') + + const hasDb = await Database.checkHasDb() + + expect(hasDb).to.equal(true) + expect(pathExistsStub.called).to.equal(false) + }) + }) + + describe('checkHasTables', () => { + it('should return true when at least one table exists', async () => { + Database.sequelize = { + getQueryInterface: () => ({ + showAllTables: sinon.stub().resolves(['users']) + }) + } + + const hasTables = await Database.checkHasTables() + + expect(hasTables).to.equal(true) + }) + + it('should return false when no tables exist', async () => { + Database.sequelize = { + getQueryInterface: () => ({ + showAllTables: sinon.stub().resolves([]) + }) + } + + const hasTables = await Database.checkHasTables() + + expect(hasTables).to.equal(false) + }) + }) + + describe('addPostgresTriggers', () => { + function captureQueries(existingTriggers = []) { + const queries = [] + Database.sequelize = { + query: async (sql) => { + queries.push(sql) + const count = existingTriggers.filter((name) => sql.includes(`tgname = '${name}'`)).length + return [[{ count }]] + } + } + return queries + } + + it('should create title and author names triggers with folded lowercase identifiers', async () => { + const queries = captureQueries() + + await Database.addPostgresTriggers() + + const functions = queries.filter((sql) => sql.includes('CREATE OR REPLACE FUNCTION')) + const triggers = queries.filter((sql) => sql.includes('CREATE TRIGGER')) + expect(functions.length).to.equal(7) + expect(triggers.length).to.equal(7) + + const allDdl = [...functions, ...triggers].join('\n') + // No camelCase identifiers may leak into postgres DDL - unquoted identifiers fold to lowercase + expect(allDdl).to.not.match(/libraryItems|bookAuthors|mediaId|titleIgnorePrefix|authorNames|bookId|authorId|lastFirst|createdAt/) + + const authorTrigger = triggers.find((sql) => sql.includes('update_library_items_author_names_on_authors_update')) + expect(authorTrigger).to.include('AFTER UPDATE OF name ON authors') + + const insertFn = functions.find((sql) => sql.includes('update_library_items_author_names_on_book_authors_insert_fn')) + expect(insertFn).to.include("string_agg(authors.name, ', ' ORDER BY bookauthors.createdat ASC)") + expect(insertFn).to.include('WHERE mediaid = NEW.bookid') + + const deleteFn = functions.find((sql) => sql.includes('update_library_items_author_names_on_book_authors_delete_fn')) + expect(deleteFn).to.include('WHERE mediaid = OLD.bookid') + }) + + it('should skip triggers that already exist', async () => { + const queries = captureQueries(['update_library_items_title']) + + await Database.addPostgresTriggers() + + const titleFn = queries.find((sql) => sql.includes('update_library_items_title_fn')) + expect(titleFn).to.equal(undefined) + const otherFns = queries.filter((sql) => sql.includes('CREATE OR REPLACE FUNCTION')) + expect(otherFns.length).to.equal(6) + }) + + it('should dispatch to postgres triggers from addTriggers', async () => { + Database.dialect = 'postgres' + const queries = captureQueries() + + await Database.addTriggers() + + expect(queries.some((sql) => sql.includes('CREATE TRIGGER'))).to.equal(true) + }) + }) +}) diff --git a/test/server/managers/BackupManager.test.js b/test/server/managers/BackupManager.test.js new file mode 100644 index 000000000..5c5797814 --- /dev/null +++ b/test/server/managers/BackupManager.test.js @@ -0,0 +1,189 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const os = require('os') +const Path = require('path') +const EventEmitter = require('events') +const childProcess = require('child_process') +const sqlite3 = require('sqlite3') + +const BackupManager = require('../../../server/managers/BackupManager') +const Backup = require('../../../server/objects/Backup') +const Database = require('../../../server/Database') + +describe('BackupManager', () => { + let originalDialect + let originalDbPath + let originalConfigPath + let originalMetadataPath + + beforeEach(() => { + originalDialect = Database.dialect + originalDbPath = Database.dbPath + originalConfigPath = global.ConfigPath + originalMetadataPath = global.MetadataPath + global.MetadataPath = os.tmpdir() + }) + + afterEach(() => { + Database.dialect = originalDialect + Database.dbPath = originalDbPath + global.ConfigPath = originalConfigPath + global.MetadataPath = originalMetadataPath + sinon.restore() + }) + + it('should select Postgres custom-format backups for the Postgres dialect', () => { + Database.dialect = 'postgres' + const manager = new BackupManager() + + expect(manager.databaseBackupConfig).to.deep.equal({ + dialect: 'postgres', + entryName: 'absdatabase.postgres.dump' + }) + }) + + it('should create Postgres dumps with pg_dump without exposing credentials in argv', async () => { + Database.dialect = 'postgres' + Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf' + global.ConfigPath = os.tmpdir() + + const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => { + callback(null, '', '') + }) + const manager = new BackupManager() + const backup = new Backup() + backup.id = '2026-08-02T0130' + + const dumpPath = await manager.backupPostgresDb(backup) + + expect(dumpPath).to.equal(Path.join(os.tmpdir(), 'absdatabase.2026-08-02T0130.postgres.dump')) + expect(execFileStub.calledOnce).to.equal(true) + expect(execFileStub.firstCall.args[0]).to.equal('pg_dump') + expect(execFileStub.firstCall.args[1]).to.deep.equal([ + '--format=custom', + '--no-owner', + '--no-acl', + '--file', + dumpPath, + '--host', + 'localhost', + '--dbname', + 'audiobookshelf', + '--port', + '5432', + '--username', + 'absuser' + ]) + expect(execFileStub.firstCall.args[1].join(' ')).to.not.include('secretpass') + expect(execFileStub.firstCall.args[2].env.PGPASSWORD).to.equal('secretpass') + expect(execFileStub.firstCall.args[2].timeout).to.be.a('number') + }) + + it('should restore Postgres dumps in one transaction and clean existing objects', async () => { + Database.dialect = 'postgres' + Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf' + + const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => { + callback(null, '', '') + }) + const manager = new BackupManager() + + await manager.restorePostgresDb('/config/absdatabase-postgres-temp.dump') + + expect(execFileStub.firstCall.args[0]).to.equal('pg_restore') + expect(execFileStub.firstCall.args[1]).to.deep.equal([ + '--clean', + '--if-exists', + '--exit-on-error', + '--single-transaction', + '--no-owner', + '--no-acl', + '/config/absdatabase-postgres-temp.dump', + '--host', + 'localhost', + '--dbname', + 'audiobookshelf', + '--port', + '5432', + '--username', + 'absuser' + ]) + expect(execFileStub.firstCall.args[1].join(' ')).to.not.include('secretpass') + expect(execFileStub.firstCall.args[2].env.PGPASSWORD).to.equal('secretpass') + }) + + it('should redact database credentials from failed pg command errors', async () => { + Database.dialect = 'postgres' + Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf' + global.ConfigPath = os.tmpdir() + + sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => { + const error = new Error('Command failed: pg_dump --dbname postgresql://absuser:secretpass@localhost/audiobookshelf\npg_dump: error: password authentication failed') + error.cmd = 'pg_dump --dbname postgresql://absuser:secretpass@localhost/audiobookshelf' + callback(error, '', 'connection using password secretpass failed') + }) + const manager = new BackupManager() + const backup = new Backup() + backup.id = '2026-08-02T0130' + + let error + try { + await manager.backupPostgresDb(backup) + } catch (caughtError) { + error = caughtError + } + + expect(error).to.be.an('error') + expect(error.message).to.not.include('secretpass') + expect(error.cmd).to.not.include('secretpass') + expect(error.stderr).to.not.include('secretpass') + expect(error.message).to.include('***') + }) + + it('should reject pg commands when DATABASE_URL is not a valid URI', async () => { + Database.dialect = 'postgres' + Database.dbPath = 'not a connection uri' + global.ConfigPath = os.tmpdir() + + const manager = new BackupManager() + const backup = new Backup() + backup.id = '2026-08-02T0130' + + let error + try { + await manager.backupPostgresDb(backup) + } catch (caughtError) { + error = caughtError + } + + expect(error).to.be.an('error') + expect(error.message).to.include('valid postgres connection URI') + }) + + it('should reject SQLite backup open errors without an uncaught sqlite event', async () => { + Database.dialect = 'sqlite' + Database.dbPath = '/config/absdatabase.sqlite' + global.ConfigPath = os.tmpdir() + + sinon.stub(sqlite3, 'Database').callsFake(function (_dbPath, callback) { + const db = new EventEmitter() + db.close = (closeCallback) => closeCallback() + process.nextTick(() => callback(Object.assign(new Error('unable to open database file'), { code: 'SQLITE_CANTOPEN' }))) + return db + }) + + const manager = new BackupManager() + const backup = new Backup() + backup.id = '2026-08-02T0130' + + let error + try { + await manager.backupSqliteDb(backup) + } catch (caughtError) { + error = caughtError + } + + expect(error).to.be.an('error') + expect(error.code).to.equal('SQLITE_CANTOPEN') + }) +}) diff --git a/test/server/managers/MigrationManager.test.js b/test/server/managers/MigrationManager.test.js index af2e9da8f..a306eb5be 100644 --- a/test/server/managers/MigrationManager.test.js +++ b/test/server/managers/MigrationManager.test.js @@ -189,6 +189,52 @@ describe('MigrationManager', () => { expect(loggerInfoStub.calledWith(sinon.match('Restored the original database'))).to.be.true expect(processExitStub.calledOnce).to.be.true }) + + it('should skip sqlite backup workflow for postgres migrations', async () => { + // Arrange + migrationManager.serverVersion = '1.2.0' + migrationManager.databaseVersion = '1.1.0' + migrationManager.maxVersion = '1.1.0' + migrationManager.initialized = true + sequelizeStub.getDialect.returns('postgres') + + umzugStub.migrations.resolves([{ name: 'v1.2.0-migration.js' }]) + umzugStub.executed.resolves([{ name: 'v1.1.0-migration.js' }]) + + // Act + await migrationManager.runMigrations() + + // Assert + expect(umzugStub.up.calledOnce).to.be.true + expect(fsCopyStub.called).to.be.false + expect(fsRemoveStub.called).to.be.false + }) + }) + + describe('tableExists', () => { + it('should use queryInterface.tableExists when available', async () => { + const tableExistsStub = sinon.stub().resolves(true) + const sequelize = sinon.createStubInstance(Sequelize) + sequelize.getQueryInterface.returns({ tableExists: tableExistsStub }) + const manager = new MigrationManager(sequelize, false, configPath) + + const exists = await manager.tableExists('migrationsMeta') + + expect(exists).to.equal(true) + expect(tableExistsStub.calledOnceWithExactly('migrationsMeta')).to.equal(true) + }) + + it('should fallback to showAllTables and support object table names', async () => { + const sequelize = sinon.createStubInstance(Sequelize) + sequelize.getQueryInterface.returns({ + showAllTables: sinon.stub().resolves([{ tableName: 'migrationsMeta' }]) + }) + const manager = new MigrationManager(sequelize, false, configPath) + + const exists = await manager.tableExists('migrationsMeta') + + expect(exists).to.equal(true) + }) }) describe('fetchVersionsFromDatabase', () => { @@ -280,6 +326,20 @@ describe('MigrationManager', () => { expect(error.message).to.equal('Database query failed') } }) + + it('should support lowercase maxversion alias from postgres drivers', async () => { + const sequelize = sinon.createStubInstance(Sequelize) + sequelize.query.onFirstCall().resolves([{ version: '1.1.0' }]) + sequelize.query.onSecondCall().resolves([{ maxversion: '1.2.0' }]) + + const manager = new MigrationManager(sequelize, false, configPath) + manager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves() + + await manager.fetchVersionsFromDatabase() + + expect(manager.databaseVersion).to.equal('1.1.0') + expect(manager.maxVersion).to.equal('1.2.0') + }) }) describe('updateMaxVersion', () => { diff --git a/test/server/migrations/v2.15.0-series-column-unique.test.js b/test/server/migrations/v2.15.0-series-column-unique.test.js index 7fce7b7a1..0439dbbb5 100644 --- a/test/server/migrations/v2.15.0-series-column-unique.test.js +++ b/test/server/migrations/v2.15.0-series-column-unique.test.js @@ -88,6 +88,23 @@ describe('migration-v2.15.0-series-column-unique', () => { await queryInterface.dropTable('Series') await queryInterface.dropTable('BookSeries') }) + + it('should skip NOCASE reindex on postgres dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + const originalQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize) + const queryStub = sinon.stub(queryInterface.sequelize, 'query').callsFake((sql, options) => { + if (sql === 'REINDEX NOCASE;') { + throw new Error('Unexpected NOCASE reindex in postgres mode') + } + return originalQuery(sql, options) + }) + + await up({ context: { queryInterface, logger: Logger } }) + + expect(queryStub.neverCalledWith('REINDEX NOCASE;')).to.be.true + expect(loggerInfoStub.calledWith(sinon.match('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.be.true + }) + it('upgrade with no duplicate series', async () => { // Add some entries to the Series table using the UUID for the ids await queryInterface.bulkInsert('Series', [ diff --git a/test/server/migrations/v2.15.1-reindex-nocase.test.js b/test/server/migrations/v2.15.1-reindex-nocase.test.js new file mode 100644 index 000000000..50fe89174 --- /dev/null +++ b/test/server/migrations/v2.15.1-reindex-nocase.test.js @@ -0,0 +1,41 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const { up, down } = require('../../../server/migrations/v2.15.1-reindex-nocase') +const Logger = require('../../../server/Logger') + +describe('migration-v2.15.1-reindex-nocase', () => { + afterEach(() => { + sinon.restore() + }) + + it('should skip reindex on non-sqlite dialect', async () => { + const queryStub = sinon.stub().resolves() + const loggerInfoStub = sinon.stub(Logger, 'info') + const queryInterface = { + sequelize: { + getDialect: () => 'postgres', + query: queryStub + } + } + + await up({ context: { queryInterface, logger: Logger } }) + + expect(queryStub.called).to.equal(false) + expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.equal(true) + }) + + it('should log no-op on down migration', async () => { + const loggerInfoStub = sinon.stub(Logger, 'info') + const queryInterface = { + sequelize: { + getDialect: () => 'postgres', + query: sinon.stub().resolves() + } + } + + await down({ context: { queryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] No action required for downgrade'))).to.equal(true) + }) +}) diff --git a/test/server/migrations/v2.15.2-index-creation.test.js b/test/server/migrations/v2.15.2-index-creation.test.js new file mode 100644 index 000000000..d9257a483 --- /dev/null +++ b/test/server/migrations/v2.15.2-index-creation.test.js @@ -0,0 +1,98 @@ +const { expect } = require('chai') +const { Sequelize, DataTypes } = require('sequelize') +const sinon = require('sinon') + +const Logger = require('../../../server/Logger') +const { up, down } = require('../../../server/migrations/v2.15.2-index-creation') + +describe('migration-v2.15.2-index-creation', () => { + let sequelize + let queryInterface + + beforeEach(async () => { + sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + queryInterface = sequelize.getQueryInterface() + + await queryInterface.createTable('bookAuthors', { + id: { type: DataTypes.INTEGER, primaryKey: true }, + authorId: { type: DataTypes.INTEGER } + }) + + await queryInterface.createTable('bookSeries', { + id: { type: DataTypes.INTEGER, primaryKey: true }, + seriesId: { type: DataTypes.INTEGER } + }) + + await queryInterface.createTable('podcastEpisodes', { + id: { type: DataTypes.INTEGER, primaryKey: true }, + createdAt: { type: DataTypes.DATE }, + podcastId: { type: DataTypes.INTEGER } + }) + }) + + afterEach(async () => { + if (sequelize) await sequelize.close() + }) + + it('up should succeed when legacy podcast index is missing', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const indexes = await queryInterface.showIndex('podcastEpisodes') + expect(indexes.some((index) => index.name === 'podcastEpisode_createdAt_podcastId')).to.equal(true) + }) + + it('down should succeed when new podcast index is missing', async () => { + await down({ context: { queryInterface, logger: Logger } }) + + const indexes = await queryInterface.showIndex('podcastEpisodes') + expect(indexes.some((index) => index.name === 'podcast_episodes_created_at')).to.equal(true) + }) + + it('up should treat index names case-insensitively', async () => { + const qi = { + showIndex: sinon.stub(), + addIndex: sinon.stub().resolves(), + removeIndex: sinon.stub().resolves() + } + + qi.showIndex.onCall(0).resolves([{ name: 'bookauthor_authorid' }]) + qi.showIndex.onCall(1).resolves([{ name: 'bookseries_seriesid' }]) + qi.showIndex.onCall(2).resolves([{ name: 'podcast_episodes_created_at' }]) + qi.showIndex.onCall(3).resolves([{ name: 'podcastepisode_createdat_podcastid' }]) + + await up({ context: { queryInterface: qi, logger: Logger } }) + + expect(qi.addIndex.called).to.equal(false) + expect(qi.removeIndex.calledOnceWithExactly('podcastEpisodes', 'podcast_episodes_created_at')).to.equal(true) + }) + + it('up should continue when addIndex reports already exists', async () => { + const makePgExistsError = (sql) => { + const err = new Error('relation already exists') + err.name = 'SequelizeDatabaseError' + err.original = { code: '42P07' } + err.sql = sql + return err + } + + const qi = { + showIndex: sinon.stub(), + addIndex: sinon.stub(), + removeIndex: sinon.stub().resolves() + } + + qi.showIndex.onCall(0).resolves([]) + qi.showIndex.onCall(1).resolves([]) + qi.showIndex.onCall(2).resolves([{ name: 'podcast_episodes_created_at' }]) + qi.showIndex.onCall(3).resolves([]) + + qi.addIndex.onCall(0).rejects(makePgExistsError('CREATE INDEX bookAuthor_authorId ON bookAuthors (authorId)')) + qi.addIndex.onCall(1).rejects(makePgExistsError('CREATE INDEX bookSeries_seriesId ON bookSeries (seriesId)')) + qi.addIndex.onCall(2).rejects(makePgExistsError('CREATE INDEX podcastEpisode_createdAt_podcastId ON podcastEpisodes (createdAt, podcastId)')) + + await up({ context: { queryInterface: qi, logger: Logger } }) + + expect(qi.addIndex.callCount).to.equal(3) + expect(qi.removeIndex.calledOnceWithExactly('podcastEpisodes', 'podcast_episodes_created_at')).to.equal(true) + }) +}) diff --git a/test/server/migrations/v2.17.3-fk-constraints.test.js b/test/server/migrations/v2.17.3-fk-constraints.test.js index 33be43ce8..975302831 100644 --- a/test/server/migrations/v2.17.3-fk-constraints.test.js +++ b/test/server/migrations/v2.17.3-fk-constraints.test.js @@ -34,6 +34,16 @@ describe('migration-v2.17.3-fk-constraints', () => { await queryInterface.dropAllTables() }) + it('should skip sqlite-specific migration on non-sqlite dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + const queryStub = sinon.stub(queryInterface.sequelize, 'query').resolves([]) + + await up({ context: { queryInterface, logger: Logger } }) + + expect(queryStub.called).to.equal(false) + expect(loggerInfoStub.calledWith(sinon.match('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect'))).to.equal(true) + }) + it('should fix table foreign key constraints', async () => { // Create tables with missing foreign key constraints: libraryItems, feeds, mediaItemShares, playbackSessions, playlistMediaItems, mediaProgresses await queryInterface.sequelize.query('CREATE TABLE `libraryItems` (`id` UUID UNIQUE PRIMARY KEY, `libraryId` UUID REFERENCES `libraries` (`id`), `libraryFolderId` UUID REFERENCES `libraryFolders` (`id`));') diff --git a/test/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.test.js b/test/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.test.js index 1662d5f98..6794a5696 100644 --- a/test/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.test.js +++ b/test/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.test.js @@ -30,11 +30,16 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { expect(logger.info.calledWith('[2.17.4 migration] UPGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] OIDC is enabled, adding authOpenIDSubfolderForRedirectURLs to server settings')).to.be.true expect(queryInterface.sequelize.query.calledTwice).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true expect( - queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = "server-settings";', { + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true + expect( + queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', { replacements: { - value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }) + value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }), + settingsKey: 'server-settings' } }) ).to.be.true @@ -49,10 +54,31 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { expect(logger.info.calledWith('[2.17.4 migration] UPGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] OIDC is not enabled, no action required')).to.be.true expect(queryInterface.sequelize.query.calledOnce).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true + expect( + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true expect(logger.info.calledWith('[2.17.4 migration] UPGRADE END: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true }) + it('should handle already-parsed object server settings', async () => { + queryInterface.sequelize.query.onFirstCall().resolves([[{ value: { authActiveAuthMethods: ['openid'] } }]]) + queryInterface.sequelize.query.onSecondCall().resolves() + + await up({ context }) + + expect(queryInterface.sequelize.query.calledTwice).to.be.true + expect( + queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', { + replacements: { + value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }), + settingsKey: 'server-settings' + } + }) + ).to.be.true + }) + it('should throw an error if server settings cannot be parsed', async () => { queryInterface.sequelize.query.onFirstCall().resolves([[{ value: 'invalid json' }]]) @@ -60,7 +86,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { await up({ context }) } catch (error) { expect(queryInterface.sequelize.query.calledOnce).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true + expect( + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true expect(logger.error.calledWith('[2.17.4 migration] Error parsing server settings:')).to.be.true expect(error).to.be.instanceOf(Error) } @@ -73,7 +103,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { await up({ context }) } catch (error) { expect(queryInterface.sequelize.query.calledOnce).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true + expect( + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true expect(logger.error.calledWith('[2.17.4 migration] Server settings not found')).to.be.true expect(error).to.be.instanceOf(Error) } @@ -90,11 +124,16 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] Removing authOpenIDSubfolderForRedirectURLs from server settings')).to.be.true expect(queryInterface.sequelize.query.calledTwice).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true expect( - queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = "server-settings";', { + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true + expect( + queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', { replacements: { - value: JSON.stringify({}) + value: JSON.stringify({}), + settingsKey: 'server-settings' } }) ).to.be.true @@ -109,7 +148,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] authOpenIDSubfolderForRedirectURLs not found in server settings, no action required')).to.be.true expect(queryInterface.sequelize.query.calledOnce).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true + expect( + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE END: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true }) }) diff --git a/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js b/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js index 5b7678567..7e291927d 100644 --- a/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js +++ b/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js @@ -47,6 +47,18 @@ describe('Migration v2.19.1-copy-title-to-library-items', () => { }) describe('up', () => { + it('should skip sqlite-specific migration on non-sqlite dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + + await up({ context: { queryInterface, logger: Logger } }) + + const table = await queryInterface.describeTable('libraryItems') + expect(table).to.have.property('id') + expect(table).to.not.have.property('title') + expect(table).to.not.have.property('titleIgnorePrefix') + expect(loggerInfoStub.calledWith(sinon.match('[2.19.1 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true + }) + it('should copy title and titleIgnorePrefix to libraryItems', async () => { await up({ context: { queryInterface, logger: Logger } }) diff --git a/test/server/migrations/v2.19.4-improve-podcast-queries.test.js b/test/server/migrations/v2.19.4-improve-podcast-queries.test.js index 0ca697d70..e385318ad 100644 --- a/test/server/migrations/v2.19.4-improve-podcast-queries.test.js +++ b/test/server/migrations/v2.19.4-improve-podcast-queries.test.js @@ -75,6 +75,18 @@ describe('Migration v2.19.4-improve-podcast-queries', () => { }) describe('up', () => { + it('should skip sqlite-specific migration on non-sqlite dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + + await up({ context: { queryInterface, logger: Logger } }) + + const podcastsTable = await queryInterface.describeTable('podcasts') + const mediaProgressesTable = await queryInterface.describeTable('mediaProgresses') + expect(podcastsTable).to.not.have.property('numEpisodes') + expect(mediaProgressesTable).to.not.have.property('podcastId') + expect(loggerInfoStub.calledWith(sinon.match('[2.19.4 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true + }) + it('should add numEpisodes column to podcasts', async () => { await up({ context: { queryInterface, logger: Logger } }) diff --git a/test/server/migrations/v2.33.0-add-discover-query-indexes.test.js b/test/server/migrations/v2.33.0-add-discover-query-indexes.test.js new file mode 100644 index 000000000..57d15675a --- /dev/null +++ b/test/server/migrations/v2.33.0-add-discover-query-indexes.test.js @@ -0,0 +1,118 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const { up, down } = require('../../../server/migrations/v2.33.0-add-discover-query-indexes') +const { Sequelize } = require('sequelize') +const Logger = require('../../../server/Logger') + +describe('migration-v2.33.0-add-discover-query-indexes', () => { + let sequelize + let queryInterface + let loggerInfoStub + + beforeEach(() => { + sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + queryInterface = sequelize.getQueryInterface() + loggerInfoStub = sinon.stub(Logger, 'info') + }) + + afterEach(() => { + sinon.restore() + }) + + describe('up', () => { + beforeEach(async () => { + await queryInterface.createTable('mediaProgresses', { + id: { type: Sequelize.UUID, primaryKey: true }, + userId: { type: Sequelize.UUID, allowNull: false }, + mediaItemId: { type: Sequelize.UUID, allowNull: false }, + isFinished: { type: Sequelize.BOOLEAN, allowNull: false }, + currentTime: { type: Sequelize.FLOAT, allowNull: false } + }) + await queryInterface.createTable('bookSeries', { + id: { type: Sequelize.UUID, primaryKey: true }, + seriesId: { type: Sequelize.UUID, allowNull: false }, + bookId: { type: Sequelize.UUID, allowNull: false } + }) + }) + + it('should add both discover query indexes', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const mediaProgressIndexes = await queryInterface.showIndex('mediaProgresses') + expect(mediaProgressIndexes.some((i) => i.name === 'media_progresses_user_item_finished_time')).to.equal(true) + const bookSeriesIndexes = await queryInterface.showIndex('bookSeries') + expect(bookSeriesIndexes.some((i) => i.name === 'book_series_series_book')).to.equal(true) + }) + + it('should not fail when the indexes already exist', async () => { + await up({ context: { queryInterface, logger: Logger } }) + await up({ context: { queryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWithMatch('index media_progresses_user_item_finished_time already exists')).to.equal(true) + expect(loggerInfoStub.calledWithMatch('index book_series_series_book already exists')).to.equal(true) + }) + + it('should detect existing indexes on postgres via pg_indexes with folded table names', async () => { + // Sequelize showIndex matches relname case-sensitively and misses folded + // lowercase postgres tables, so the migration must query pg_indexes instead + const queries = [] + const fakeQueryInterface = { + sequelize: { + getDialect: () => 'postgres', + query: async (sql, options) => { + queries.push({ sql, options }) + return [[{ name: 'media_progresses_user_item_finished_time' }, { name: 'book_series_series_book' }]] + } + }, + addIndex: async () => { + throw new Error('addIndex must not be called for existing indexes') + }, + showIndex: async () => { + throw new Error('showIndex must not be used on postgres') + } + } + + await up({ context: { queryInterface: fakeQueryInterface, logger: Logger } }) + + expect(queries.length).to.be.greaterThan(0) + expect(queries.every((q) => q.sql.includes('pg_indexes'))).to.equal(true) + expect(queries.map((q) => q.options.bind[0])).to.include.members(['mediaprogresses', 'bookseries']) + expect(loggerInfoStub.calledWithMatch('index media_progresses_user_item_finished_time already exists')).to.equal(true) + expect(loggerInfoStub.calledWithMatch('index book_series_series_book already exists')).to.equal(true) + }) + }) + + describe('down', () => { + beforeEach(async () => { + await queryInterface.createTable('mediaProgresses', { + id: { type: Sequelize.UUID, primaryKey: true }, + userId: { type: Sequelize.UUID, allowNull: false }, + mediaItemId: { type: Sequelize.UUID, allowNull: false }, + isFinished: { type: Sequelize.BOOLEAN, allowNull: false }, + currentTime: { type: Sequelize.FLOAT, allowNull: false } + }) + await queryInterface.createTable('bookSeries', { + id: { type: Sequelize.UUID, primaryKey: true }, + seriesId: { type: Sequelize.UUID, allowNull: false }, + bookId: { type: Sequelize.UUID, allowNull: false } + }) + await up({ context: { queryInterface, logger: Logger } }) + }) + + it('should remove both discover query indexes', async () => { + await down({ context: { queryInterface, logger: Logger } }) + + const mediaProgressIndexes = await queryInterface.showIndex('mediaProgresses') + expect(mediaProgressIndexes.some((i) => i.name === 'media_progresses_user_item_finished_time')).to.equal(false) + const bookSeriesIndexes = await queryInterface.showIndex('bookSeries') + expect(bookSeriesIndexes.some((i) => i.name === 'book_series_series_book')).to.equal(false) + }) + + it('should not fail when the indexes do not exist', async () => { + await down({ context: { queryInterface, logger: Logger } }) + await down({ context: { queryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWithMatch('index media_progresses_user_item_finished_time does not exist')).to.equal(true) + }) + }) +}) diff --git a/test/server/migrations/v2.35.0-add-last-refresh-token.test.js b/test/server/migrations/v2.35.0-add-last-refresh-token.test.js new file mode 100644 index 000000000..7074c9577 --- /dev/null +++ b/test/server/migrations/v2.35.0-add-last-refresh-token.test.js @@ -0,0 +1,98 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const { up, down } = require('../../../server/migrations/v2.35.0-add-last-refresh-token') +const { Sequelize } = require('sequelize') +const Logger = require('../../../server/Logger') + +describe('migration-v2.35.0-add-last-refresh-token', () => { + let sequelize + let queryInterface + let loggerInfoStub + + beforeEach(() => { + sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + queryInterface = sequelize.getQueryInterface() + loggerInfoStub = sinon.stub(Logger, 'info') + }) + + afterEach(() => { + sinon.restore() + }) + + describe('up', () => { + beforeEach(async () => { + await queryInterface.createTable('sessions', { + id: { type: Sequelize.UUID, primaryKey: true }, + createdAt: { type: Sequelize.DATE, allowNull: false }, + updatedAt: { type: Sequelize.DATE, allowNull: false } + }) + }) + + it('should add lastRefreshToken columns when they do not exist', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const tableDescription = await queryInterface.describeTable('sessions') + expect(tableDescription.lastRefreshToken).to.exist + expect(tableDescription.lastRefreshTokenExpiresAt).to.exist + }) + + it('should not fail when the columns already exist', async () => { + await up({ context: { queryInterface, logger: Logger } }) + await up({ context: { queryInterface, logger: Logger } }) + + const tableDescription = await queryInterface.describeTable('sessions') + expect(tableDescription.lastRefreshToken).to.exist + expect(tableDescription.lastRefreshTokenExpiresAt).to.exist + expect(loggerInfoStub.calledWithMatch('lastRefreshToken column already exists')).to.equal(true) + }) + + it('should detect existing columns case-insensitively for postgres identifier folding', async () => { + // Postgres folds unquoted identifiers to lowercase, so describeTable on a + // migrated postgres database returns lowercase column names + const fakeQueryInterface = { + sequelize, + tableExists: async () => true, + describeTable: async () => ({ + id: {}, + lastrefreshtoken: {}, + lastrefreshtokenexpiresat: {} + }), + addColumn: async () => { + throw new Error('addColumn must not be called for existing lowercase columns') + } + } + + await up({ context: { queryInterface: fakeQueryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWithMatch('lastRefreshToken column already exists')).to.equal(true) + expect(loggerInfoStub.calledWithMatch('lastRefreshTokenExpiresAt column already exists')).to.equal(true) + }) + }) + + describe('down', () => { + beforeEach(async () => { + await queryInterface.createTable('sessions', { + id: { type: Sequelize.UUID, primaryKey: true }, + lastRefreshToken: { type: Sequelize.STRING, allowNull: true }, + lastRefreshTokenExpiresAt: { type: Sequelize.DATE, allowNull: true }, + createdAt: { type: Sequelize.DATE, allowNull: false }, + updatedAt: { type: Sequelize.DATE, allowNull: false } + }) + }) + + it('should remove lastRefreshToken columns when they exist', async () => { + await down({ context: { queryInterface, logger: Logger } }) + + const tableDescription = await queryInterface.describeTable('sessions') + expect(tableDescription.lastRefreshToken).to.not.exist + expect(tableDescription.lastRefreshTokenExpiresAt).to.not.exist + }) + + it('should not fail when the columns do not exist', async () => { + await down({ context: { queryInterface, logger: Logger } }) + await down({ context: { queryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWithMatch('lastRefreshToken column does not exist')).to.equal(true) + }) + }) +}) diff --git a/test/server/models/MediaProgress.test.js b/test/server/models/MediaProgress.test.js new file mode 100644 index 000000000..a7c490306 --- /dev/null +++ b/test/server/models/MediaProgress.test.js @@ -0,0 +1,70 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Logger = require('../../../server/Logger') +const MediaProgress = require('../../../server/models/MediaProgress') + +function createProgressInstance() { + const progress = Object.create(MediaProgress.prototype) + + Object.defineProperties(progress, { + id: { value: 'progress-1', writable: true, configurable: true }, + mediaItemId: { value: 'media-1', writable: true, configurable: true }, + duration: { value: 3600, writable: true, configurable: true }, + currentTime: { value: 120, writable: true, configurable: true }, + isFinished: { value: false, writable: true, configurable: true }, + hideFromContinueListening: { value: false, writable: true, configurable: true }, + extraData: { value: {}, writable: true, configurable: true } + }) + + progress.changed = sinon.stub().returns(false) + progress.set = sinon.stub().callsFake((payload) => Object.assign(progress, payload)) + progress.save = sinon.stub().resolves() + progress.reload = sinon.stub().resolves() + progress.constructor = { + update: sinon.stub().resolves(), + sequelize: { + escape: (value) => `'${value.toISOString ? value.toISOString() : value}'` + } + } + + return progress +} + +describe('MediaProgress', () => { + afterEach(() => { + sinon.restore() + }) + + describe('applyProgressUpdate', () => { + it('should update updatedAt via model update for valid lastUpdate', async () => { + const progress = createProgressInstance() + const infoStub = sinon.stub(Logger, 'info') + const lastUpdate = '2026-03-03T01:00:00.000Z' + + await progress.applyProgressUpdate({ currentTime: 130, lastUpdate }) + + expect(progress.save.calledOnce).to.equal(true) + expect(progress.constructor.update.calledOnce).to.equal(true) + expect(progress.constructor.update.firstCall.args[0].updatedAt.toISOString()).to.equal(new Date(lastUpdate).toISOString()) + expect(progress.constructor.update.firstCall.args[1]).to.deep.equal({ + where: { id: 'progress-1' }, + silent: true + }) + expect(progress.reload.calledOnce).to.equal(true) + expect(infoStub.calledWithMatch('[MediaProgress] Manually setting updatedAt')).to.equal(true) + }) + + it('should skip manual updatedAt update when lastUpdate is invalid', async () => { + const progress = createProgressInstance() + const warnStub = sinon.stub(Logger, 'warn') + + await progress.applyProgressUpdate({ currentTime: 130, lastUpdate: 'invalid-date' }) + + expect(progress.save.calledOnce).to.equal(true) + expect(progress.constructor.update.called).to.equal(false) + expect(progress.reload.called).to.equal(false) + expect(warnStub.calledWithMatch('[MediaProgress] Invalid date provided for lastUpdate')).to.equal(true) + }) + }) +}) diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js new file mode 100644 index 000000000..be317eb44 --- /dev/null +++ b/test/server/models/User.test.js @@ -0,0 +1,170 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const User = require('../../../server/models/User') + +describe('User model', () => { + describe('case-insensitive lookup helpers', () => { + afterEach(() => { + sinon.restore() + }) + + it('should query usernames case-insensitively', async () => { + User.sequelize = { + models: { + mediaProgress: {} + } + } + + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + await User.getUserByUsername('Madison') + + expect(findOneStub.calledOnce).to.equal(true) + const options = findOneStub.firstCall.args[0] + expect(options.where.attribute.fn).to.equal('LOWER') + expect(options.where.attribute.args[0].col).to.equal('username') + expect(options.where.logic).to.equal('madison') + }) + + it('should query emails case-insensitively', async () => { + User.sequelize = { + models: { + mediaProgress: {} + } + } + + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + await User.getUserByEmail('Example.User@Example.com') + + expect(findOneStub.calledOnce).to.equal(true) + const options = findOneStub.firstCall.args[0] + expect(options.where.attribute.fn).to.equal('LOWER') + expect(options.where.attribute.args[0].col).to.equal('email') + expect(options.where.logic).to.equal('example.user@example.com') + }) + + it('should hit the user cache for mixed-case username lookups', async () => { + User.sequelize = { + models: { + mediaProgress: {} + } + } + const cachedUser = { id: 'cache-test-user-1', username: 'CacheTestUser', email: 'cachetest@example.com', extraData: {} } + const findOneStub = sinon.stub(User, 'findOne').resolves(cachedUser) + + await User.getUserByUsername('CacheTestUser') + await User.getUserByUsername('cachetestuser') + await User.getUserByUsername('CACHETESTUSER') + + expect(findOneStub.callCount).to.equal(1) + }) + + it('should hit the user cache for mixed-case email lookups', async () => { + User.sequelize = { + models: { + mediaProgress: {} + } + } + const cachedUser = { id: 'cache-test-user-2', username: 'CacheEmailUser', email: 'CacheMail@Example.com', extraData: {} } + const findOneStub = sinon.stub(User, 'findOne').resolves(cachedUser) + + await User.getUserByEmail('CacheMail@Example.com') + await User.getUserByEmail('cachemail@example.com') + await User.getUserByEmail('CACHEMAIL@EXAMPLE.COM') + + expect(findOneStub.callCount).to.equal(1) + }) + }) + + describe('getUserByIdOrOldId', () => { + let originalSequelize + + beforeEach(() => { + originalSequelize = User.sequelize + }) + + afterEach(() => { + User.sequelize = originalSequelize + sinon.restore() + }) + + it('should resolve UUID ids via primary-key lookup on postgres', async () => { + User.sequelize = { + getDialect: () => 'postgres', + models: { + mediaProgress: {} + } + } + + const user = { id: 'e8e677b2-da16-4220-ab67-443b7714caf9' } + const findByPkStub = sinon.stub(User, 'findByPk').resolves(user) + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + const result = await User.getUserByIdOrOldId('e8e677b2-da16-4220-ab67-443b7714caf9') + + expect(result).to.equal(user) + expect(findByPkStub.calledOnce).to.equal(true) + expect(findOneStub.called).to.equal(false) + }) + + it('should resolve uppercase UUID ids via primary-key lookup on postgres', async () => { + User.sequelize = { + getDialect: () => 'postgres', + models: { + mediaProgress: {} + } + } + + const uppercaseUuid = 'E8E677B2-DA16-4220-AB67-443B7714CAF9' + const user = { id: uppercaseUuid } + const findByPkStub = sinon.stub(User, 'findByPk').resolves(user) + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + const result = await User.getUserByIdOrOldId(uppercaseUuid) + + expect(result).to.equal(user) + expect(findByPkStub.calledOnceWithExactly(uppercaseUuid, { include: User.sequelize.models.mediaProgress })).to.equal(true) + expect(findOneStub.called).to.equal(false) + }) + + it('should query legacy oldUserId with postgres-safe JSON matcher', async () => { + User.sequelize = { + getDialect: () => 'postgres', + models: { + mediaProgress: {} + } + } + + const findByPkStub = sinon.stub(User, 'findByPk').resolves(null) + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + await User.getUserByIdOrOldId('root') + + expect(findByPkStub.called).to.equal(false) + expect(findOneStub.calledOnce).to.equal(true) + + const options = findOneStub.firstCall.args[0] + expect(options.where.attribute.val).to.equal("extradata#>>'{oldUserId}'") + expect(options.where.logic).to.equal('root') + }) + + it('should keep sqlite oldUserId matcher unchanged', async () => { + User.sequelize = { + getDialect: () => 'sqlite', + models: { + mediaProgress: {} + } + } + + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + await User.getUserByIdOrOldId('root') + + expect(findOneStub.calledOnce).to.equal(true) + + const options = findOneStub.firstCall.args[0] + expect(options.where).to.deep.equal({ 'extraData.oldUserId': 'root' }) + }) + }) +}) diff --git a/test/server/scripts/migrateSqliteToPostgres.test.js b/test/server/scripts/migrateSqliteToPostgres.test.js new file mode 100644 index 000000000..f3049ab62 --- /dev/null +++ b/test/server/scripts/migrateSqliteToPostgres.test.js @@ -0,0 +1,149 @@ +const { expect } = require('chai') +const sqlite3 = require('sqlite3') + +const { + normalizeJson, + isIntegerCompatible, + convertValue, + findOverlongVarcharValues, + findIntegerTypeIssues +} = require('../../../server/scripts/migrateSqliteToPostgres') + +function openMemoryDb() { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(':memory:', (error) => { + if (error) return reject(error) + resolve(db) + }) + }) +} + +function run(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.run(sql, params, (error) => { + if (error) return reject(error) + resolve() + }) + }) +} + +function close(db) { + return new Promise((resolve, reject) => { + db.close((error) => { + if (error) return reject(error) + resolve() + }) + }) +} + +describe('migrateSqliteToPostgres script helpers', () => { + it('should keep malformed JSON payloads insertable by returning valid JSON text', () => { + const malformed = '"{"x",1}"' + const normalized = normalizeJson(malformed) + + expect(() => JSON.parse(normalized)).to.not.throw() + expect(JSON.parse(normalized)).to.equal('"{"x",1}"') + }) + + it('should detect overlong varchar values before migration', async () => { + const db = await openMemoryDb() + + try { + await run(db, 'CREATE TABLE books (subtitle TEXT)') + await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['x'.repeat(300)]) + await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['ok']) + + const tablesToMigrate = [{ sqliteTable: 'books', postgresTable: 'books' }] + const pgColumnsByTable = new Map([ + [ + 'books', + new Map([ + [ + 'subtitle', + { + column_name: 'subtitle', + data_type: 'character varying', + character_maximum_length: 255 + } + ] + ]) + ] + ]) + + const issues = await findOverlongVarcharValues(db, tablesToMigrate, pgColumnsByTable) + + expect(issues).to.have.length(1) + expect(issues[0]).to.include({ + sqliteTable: 'books', + sqliteColumn: 'subtitle', + postgresTable: 'books', + postgresColumn: 'subtitle', + maxLength: 255, + actualMaxLength: 300, + overCount: 1 + }) + } finally { + await close(db) + } + }) + + it('should detect non-integer values for integer target columns', async () => { + const db = await openMemoryDb() + + try { + await run(db, 'CREATE TABLE playbackSessions (timeListening REAL)') + await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [25.802536999999997]) + await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [42]) + + const tablesToMigrate = [{ sqliteTable: 'playbackSessions', postgresTable: 'playbackSessions' }] + const pgColumnsByTable = new Map([ + [ + 'playbackSessions', + new Map([ + [ + 'timelistening', + { + column_name: 'timeListening', + data_type: 'integer' + } + ] + ]) + ] + ]) + + const issues = await findIntegerTypeIssues(db, tablesToMigrate, pgColumnsByTable) + + expect(issues).to.have.length(1) + expect(issues[0].sqliteTable).to.equal('playbackSessions') + expect(issues[0].sqliteColumn).to.equal('timeListening') + expect(issues[0].postgresType).to.equal('integer') + expect(issues[0].badCount).to.equal(1) + } finally { + await close(db) + } + }) + + it('should only accept integer-compatible values for integer columns', () => { + expect(isIntegerCompatible(10)).to.equal(true) + expect(isIntegerCompatible('10')).to.equal(true) + expect(isIntegerCompatible(10.5)).to.equal(false) + expect(isIntegerCompatible('10.5')).to.equal(false) + expect(isIntegerCompatible('32768', 'smallint')).to.equal(false) + expect(isIntegerCompatible('9007199254740993', 'bigint')).to.equal(true) + }) + + it('should coerce integer-like strings for postgres integer columns', () => { + expect(convertValue('42', { data_type: 'integer' })).to.equal(42) + expect(convertValue('-7', { data_type: 'bigint' })).to.equal('-7') + expect(convertValue('9007199254740993', { data_type: 'bigint' })).to.equal('9007199254740993') + expect(convertValue('4.2', { data_type: 'integer' })).to.equal('4.2') + }) + + it('should always return valid json text for postgres json columns', () => { + const convertedObject = convertValue({ a: 1 }, { data_type: 'jsonb', udt_name: 'jsonb' }) + const convertedMalformed = convertValue('"{"x",1}"', { data_type: 'jsonb', udt_name: 'jsonb' }) + + expect(JSON.parse(convertedObject)).to.deep.equal({ a: 1 }) + expect(JSON.parse(convertedMalformed)).to.equal('"{"x",1}"') + }) +}) diff --git a/test/server/utils/queries/libraryFilters.test.js b/test/server/utils/queries/libraryFilters.test.js new file mode 100644 index 000000000..f956997ce --- /dev/null +++ b/test/server/utils/queries/libraryFilters.test.js @@ -0,0 +1,96 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Database = require('../../../../server/Database') +const Logger = require('../../../../server/Logger') +const libraryFilters = require('../../../../server/utils/queries/libraryFilters') +const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters') +const libraryItemsPodcastFilters = require('../../../../server/utils/queries/libraryItemsPodcastFilters') + +describe('libraryFilters shelf resilience', () => { + afterEach(() => { + sinon.restore() + }) + + it('should return empty discover shelf when discover query fails', async () => { + sinon.stub(Database, 'isPostgresDialect').returns(true) + sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').rejects(new Error('discover failed')) + const errorStub = sinon.stub(Logger, 'error') + + const result = await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10) + + expect(result).to.deep.equal({ libraryItems: [], count: 0 }) + expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load discover shelf for library "library-1"')).to.equal(true) + }) + + it('should return empty in-progress shelf when a query fails', async () => { + sinon.stub(Database, 'isPostgresDialect').returns(true) + sinon.stub(libraryItemsBookFilters, 'getFilteredLibraryItems').rejects(new Error('progress failed')) + const errorStub = sinon.stub(Logger, 'error') + + const result = await libraryFilters.getMediaItemsInProgress({ isBook: true, id: 'library-1' }, { id: 'user-1' }, [], 10) + + expect(result).to.deep.equal({ items: [], count: 0 }) + expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load in-progress shelf for library "library-1"')).to.equal(true) + }) + + it('should return empty continue-series shelf when a query fails', async () => { + sinon.stub(Database, 'isPostgresDialect').returns(true) + sinon.stub(libraryItemsBookFilters, 'getContinueSeriesLibraryItems').rejects(new Error('continue failed')) + const errorStub = sinon.stub(Logger, 'error') + + const result = await libraryFilters.getLibraryItemsContinueSeries({ id: 'library-1' }, { id: 'user-1' }, [], 10) + + expect(result).to.deep.equal({ libraryItems: [], count: 0 }) + expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load continue-series shelf for library "library-1"')).to.equal(true) + }) + + it('should return empty newest podcast episodes shelf when a query fails', async () => { + sinon.stub(Database, 'isPostgresDialect').returns(true) + sinon.stub(libraryItemsPodcastFilters, 'getFilteredPodcastEpisodes').rejects(new Error('podcast failed')) + const errorStub = sinon.stub(Logger, 'error') + + const result = await libraryFilters.getNewestPodcastEpisodes({ mediaType: 'podcast', id: 'library-1' }, { id: 'user-1' }, 10) + + expect(result).to.deep.equal({ libraryItems: [], count: 0 }) + expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load newest-podcast-episodes shelf for library "library-1"')).to.equal(true) + }) + + it('should rethrow shelf query errors on sqlite to keep upstream behavior', async () => { + sinon.stub(Database, 'isPostgresDialect').returns(false) + sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').rejects(new Error('discover failed')) + + let error + try { + await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10) + } catch (caughtError) { + error = caughtError + } + + expect(error).to.be.an('error') + expect(error.message).to.equal('discover failed') + }) + + it('should keep discover shelf mapping behavior when query succeeds', async () => { + const libraryItem = { + toOldJSONMinified: () => ({ id: 'item-1' }), + rssFeed: { toOldJSONMinified: () => ({ id: 'rss-1' }) }, + mediaItemShare: { id: 'share-1' } + } + sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').resolves({ + libraryItems: [libraryItem], + count: 1 + }) + + const result = await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10) + + expect(result.count).to.equal(1) + expect(result.libraryItems).to.deep.equal([ + { + id: 'item-1', + rssFeed: { id: 'rss-1' }, + mediaItemShare: { id: 'share-1' } + } + ]) + }) +}) diff --git a/test/server/utils/queries/libraryItemsBookFilters.test.js b/test/server/utils/queries/libraryItemsBookFilters.test.js new file mode 100644 index 000000000..9be7baff4 --- /dev/null +++ b/test/server/utils/queries/libraryItemsBookFilters.test.js @@ -0,0 +1,148 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const Sequelize = require('sequelize') + +const Database = require('../../../../server/Database') +const Logger = require('../../../../server/Logger') +const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters') + +function createSequelizeStub(dialect, models) { + return { + getDialect: () => dialect, + escape: (value) => `'${String(value).replace(/'/g, "''")}'`, + random: () => ({ fn: 'random' }), + models + } +} + +function createModelStubs() { + return { + book: { + findAndCountAll: sinon.stub(), + findAll: sinon.stub(), + count: sinon.stub() + }, + libraryItem: {}, + feed: {}, + bookSeries: {}, + series: { + findAll: sinon.stub() + }, + bookAuthor: {}, + author: {}, + mediaProgress: {}, + mediaItemShare: {} + } +} + +describe('libraryItemsBookFilters postgres query safety', () => { + let originalSequelize + let originalServerSettings + let modelStubs + + beforeEach(() => { + originalSequelize = Database.sequelize + originalServerSettings = global.ServerSettings + modelStubs = createModelStubs() + global.ServerSettings = { + sortingIgnorePrefix: false + } + }) + + afterEach(() => { + Database.sequelize = originalSequelize + global.ServerSettings = originalServerSettings + sinon.restore() + }) + + it('should avoid generating IN () when collapse-series has no include ids', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 }) + + sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({ + booksToExclude: [], + bookSeriesToInclude: [] + }) + const debugStub = sinon.stub(Logger, 'debug') + + await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, true, [], 20, 0) + + const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0] + const displayTitleExpression = findOptions.attributes.include[0][0].val + + expect(displayTitleExpression).to.include('COALESCE(NULL, libraryItem.title)') + expect(displayTitleExpression).to.not.include('IN ()') + expect(debugStub.calledWithMatch('collapse-series produced no include IDs')).to.equal(true) + }) + + it('should escape collapse-series ids safely for postgres subquery', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 }) + + sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({ + booksToExclude: [], + bookSeriesToInclude: [{ id: "series-id-'quoted'", numBooks: 2, libraryItemIds: [] }] + }) + global.ServerSettings.sortingIgnorePrefix = true + + await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.title', false, true, [], 20, 0) + + const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0] + const displayTitleExpression = findOptions.attributes.include[0][0].val + + expect(displayTitleExpression).to.include("bs.id IN ('series-id-''quoted''')") + expect(displayTitleExpression).to.include('libraryItem.titleIgnorePrefix') + }) + + it('should use postgres-safe join alias for sequence sorting', () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + + const order = libraryItemsBookFilters.getOrder('sequence', false, false) + const expression = order[0][0].val + + expect(expression).to.include('"series->bookSeries"."sequence"') + expect(expression).to.include('CASE WHEN BTRIM') + }) + + it('should log generated query when findAndCountAll fails', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.book.findAndCountAll.rejects(new Error('boom')) + const errorStub = sinon.stub(Logger, 'error') + + try { + await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, false, [], 20, 0) + expect.fail('Expected getFilteredLibraryItems to throw') + } catch (error) { + expect(error.message).to.equal('boom') + } + + expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll failed: boom')).to.equal(true) + expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll query:')).to.equal(true) + }) + + it('should use postgres-safe books->bookSeries alias in collapse-series selector query', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.series.findAll.resolves([]) + + await libraryItemsBookFilters.getCollapseSeriesBooksToExclude({ where: {}, include: [] }, null) + + const findAllOptions = modelStubs.series.findAll.firstCall.args[0] + const orderExpression = findAllOptions.order[0].val + + expect(orderExpression).to.include('"books->bookSeries"."sequence"') + }) + + it('should use boolean false for discover not-started media progress on postgres', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.series.findAll.resolves([]) + modelStubs.book.count.resolves(0) + modelStubs.book.findAll.resolves([]) + + await libraryItemsBookFilters.getDiscoverLibraryItems('library-1', { id: 'user-1', canAccessExplicitContent: true }, [], 10) + + const countOptions = modelStubs.book.count.firstCall.args[0] + const progressFilter = countOptions.where[0]['$mediaProgresses.isFinished$'] + + expect(progressFilter[Sequelize.Op.or]).to.deep.equal([null, false]) + }) +}) diff --git a/test/server/utils/queries/libraryItemsPodcastFilters.test.js b/test/server/utils/queries/libraryItemsPodcastFilters.test.js new file mode 100644 index 000000000..79938abdc --- /dev/null +++ b/test/server/utils/queries/libraryItemsPodcastFilters.test.js @@ -0,0 +1,75 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Database = require('../../../../server/Database') +const libraryItemsPodcastFilters = require('../../../../server/utils/queries/libraryItemsPodcastFilters') + +describe('libraryItemsPodcastFilters dialect behavior', () => { + let originalSequelize + let originalServerSettings + + beforeEach(() => { + originalSequelize = Database.sequelize + originalServerSettings = global.ServerSettings + global.ServerSettings = { sortingIgnorePrefix: false } + }) + + afterEach(() => { + Database.sequelize = originalSequelize + global.ServerSettings = originalServerSettings + sinon.restore() + }) + + it('should build postgres json tag permission predicate', () => { + Database.sequelize = { getDialect: () => 'postgres' } + + const { podcastWhere, replacements } = libraryItemsPodcastFilters.getUserPermissionPodcastWhereQuery({ + canAccessExplicitContent: true, + permissions: { + accessAllTags: false, + itemTagsSelected: ['fiction'], + selectedTagsNotAccessible: false + } + }) + + expect(replacements.userTagsSelected).to.deep.equal(['fiction']) + expect(podcastWhere[0].attribute.val).to.include('jsonb_array_elements_text') + }) + + it('should build sqlite no-case author sort expression', () => { + Database.sequelize = { getDialect: () => 'sqlite' } + + const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false) + + expect(order[0][0].val).to.include('podcast.author COLLATE NOCASE') + }) + + it('should build postgres lower author sort expression', () => { + Database.sequelize = { getDialect: () => 'postgres' } + + const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false) + + expect(order[0][0].val).to.include('LOWER(podcast.author)') + }) + + it('should use postgres json duration extraction in podcast stats query', async () => { + const queryStub = sinon.stub() + queryStub.onFirstCall().resolves([[{ totalSize: 1000 }]]) + queryStub.onSecondCall().resolves([[{ totalDuration: '12.3', totalItems: '4', numAudioFiles: '9' }]]) + + Database.sequelize = { + getDialect: () => 'postgres', + query: queryStub + } + + const result = await libraryItemsPodcastFilters.getPodcastLibraryStats('library-1') + + expect(queryStub.secondCall.args[0]).to.include('NULLIF(pe.audioFile::jsonb #>>') + expect(result).to.deep.equal({ + totalSize: 1000, + totalDuration: '12.3', + numAudioFiles: '9', + totalItems: '4' + }) + }) +}) diff --git a/test/server/utils/queries/seriesFilters.test.js b/test/server/utils/queries/seriesFilters.test.js new file mode 100644 index 000000000..eb1c3cb54 --- /dev/null +++ b/test/server/utils/queries/seriesFilters.test.js @@ -0,0 +1,107 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Database = require('../../../../server/Database') +const seriesFilters = require('../../../../server/utils/queries/seriesFilters') +const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters') + +function encodedFilter(group, value) { + return `${group}.${encodeURIComponent(Buffer.from(value).toString('base64'))}` +} + +describe('seriesFilters dialect behavior', () => { + let originalSequelize + let originalServerSettings + let modelStubs + + function setDialect(dialect) { + Database.sequelize = { + getDialect: () => dialect, + random: sinon.stub(), + models: modelStubs + } + } + + beforeEach(() => { + originalSequelize = Database.sequelize + originalServerSettings = global.ServerSettings + modelStubs = { + series: { + findAndCountAll: sinon.stub().resolves({ rows: [], count: 0 }) + }, + bookSeries: {}, + book: {}, + libraryItem: {}, + author: {}, + feed: {} + } + + global.ServerSettings = { sortingIgnorePrefix: false } + sinon.stub(libraryItemsBookFilters, 'getUserPermissionBookWhereQuery').returns({ + bookWhere: [], + replacements: {} + }) + }) + + afterEach(() => { + Database.sequelize = originalSequelize + global.ServerSettings = originalServerSettings + sinon.restore() + }) + + it('should build postgres progress filter with TRUE and FALSE literals', async () => { + setDialect('postgres') + + await seriesFilters.getFilteredSeries( + { id: 'library-1', settings: { hideSingleBookSeries: false } }, + { id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } }, + encodedFilter('progress', 'not-started'), + 'name', + false, + [], + 10, + 0 + ) + + const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0] + const progressWhere = findOptions.where[1] + + expect(progressWhere.attribute.val).to.include('mp.isFinished = TRUE') + }) + + it('should build sqlite no-case name sort expression', async () => { + setDialect('sqlite') + + await seriesFilters.getFilteredSeries( + { id: 'library-1', settings: { hideSingleBookSeries: false } }, + { id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } }, + null, + 'name', + false, + [], + 10, + 0 + ) + + const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0] + expect(findOptions.order[0][0].val).to.equal('series.name COLLATE NOCASE') + }) + + it('should build postgres lower-case name sort expression', async () => { + setDialect('postgres') + + await seriesFilters.getFilteredSeries( + { id: 'library-1', settings: { hideSingleBookSeries: false } }, + { id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } }, + null, + 'name', + false, + [], + 10, + 0 + ) + + const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0] + expect(findOptions.order[0][0].val).to.equal('LOWER(series.name)') + }) +}) diff --git a/test/server/utils/queries/sqlBooleanSafety.test.js b/test/server/utils/queries/sqlBooleanSafety.test.js new file mode 100644 index 000000000..c70704682 --- /dev/null +++ b/test/server/utils/queries/sqlBooleanSafety.test.js @@ -0,0 +1,47 @@ +const { expect } = require('chai') +const fs = require('fs') +const path = require('path') + +function getJsFiles(dirPath) { + const entries = fs.readdirSync(dirPath, { withFileTypes: true }) + return entries.flatMap((entry) => { + const entryPath = path.join(dirPath, entry.name) + if (entry.isDirectory()) return getJsFiles(entryPath) + if (entry.isFile() && entry.name.endsWith('.js')) return [entryPath] + return [] + }) +} + +describe('query boolean safety', () => { + it('should not use numeric literals for SQL boolean fields in query code', () => { + const queriesDir = path.resolve(__dirname, '../../../../server/utils/queries') + const files = getJsFiles(queriesDir) + const offenders = [] + const forbiddenPatterns = [ + { + regex: /\$mediaProgresses\.isFinished\$[\s\S]{0,120}\[null,\s*[01]\]/g, + message: 'numeric literal in mediaProgresses.isFinished filter' + }, + { + regex: /\$books\.mediaProgresses\.isFinished\$[\s\S]{0,120}\[null,\s*[01]\]/g, + message: 'numeric literal in books.mediaProgresses.isFinished filter' + }, + { + regex: /\b(?:mp|mediaProgresses|b)\.(?:isFinished|explicit|abridged)\s*=\s*[01]\b/g, + message: 'direct SQL boolean comparison against 0/1' + } + ] + + files.forEach((filePath) => { + const content = fs.readFileSync(filePath, 'utf8') + forbiddenPatterns.forEach(({ regex, message }) => { + const matches = [...content.matchAll(regex)] + matches.forEach((match) => { + offenders.push(`${path.relative(queriesDir, filePath)}: ${message}: ${match[0].replace(/\s+/g, ' ').trim()}`) + }) + }) + }) + + expect(offenders).to.deep.equal([]) + }) +}) diff --git a/test/server/utils/sqlDialectHelpers.test.js b/test/server/utils/sqlDialectHelpers.test.js new file mode 100644 index 000000000..82d9154d8 --- /dev/null +++ b/test/server/utils/sqlDialectHelpers.test.js @@ -0,0 +1,68 @@ +const { expect } = require('chai') + +const { + booleanLiteral, + noCaseSortExpression, + coalesceFunctionName, + jsonArrayContainsAny, + jsonArrayContainsValue, + jsonArrayExpand, + jsonPathText, + jsonPathNumber, + safeTextToDoubleExpression, + safeTextToIntegerExpression +} = require('../../../server/utils/sqlDialectHelpers') + +const sqlite = { + getDialect: () => 'sqlite' +} + +const postgres = { + getDialect: () => 'postgres' +} + +describe('sqlDialectHelpers', () => { + it('should return sqlite and postgres boolean literals', () => { + expect(booleanLiteral(true, sqlite)).to.equal('1') + expect(booleanLiteral(false, sqlite)).to.equal('0') + expect(booleanLiteral(true, postgres)).to.equal('TRUE') + expect(booleanLiteral(false, postgres)).to.equal('FALSE') + }) + + it('should generate case-insensitive sort expressions', () => { + expect(noCaseSortExpression('name', sqlite)).to.equal('name COLLATE NOCASE') + expect(noCaseSortExpression('name', postgres)).to.equal('LOWER(name)') + }) + + it('should choose the correct null-coalescing function name', () => { + expect(coalesceFunctionName(sqlite)).to.equal('IFNULL') + expect(coalesceFunctionName(postgres)).to.equal('COALESCE') + }) + + it('should generate array membership count queries per dialect', () => { + expect(jsonArrayContainsAny('tags', 'selectedTags', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:selectedTags))') + expect(jsonArrayContainsAny('tags', 'selectedTags', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:selectedTags))") + + expect(jsonArrayContainsValue('tags', 'tag', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value = :tag)') + expect(jsonArrayContainsValue('tags', 'tag', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :tag)") + }) + + it('should generate json array expansion and path extraction by dialect', () => { + expect(jsonArrayExpand('books.tags', sqlite)).to.equal('json_each(books.tags)') + expect(jsonArrayExpand('books.tags', postgres)).to.equal("jsonb_array_elements_text(COALESCE(books.tags::jsonb, '[]'::jsonb)) AS json_each(value)") + + expect(jsonPathText('payload', ['metadata', 'filename'], sqlite)).to.equal("json_extract(payload, '$.metadata.filename')") + expect(jsonPathText('payload', ['metadata', 'filename'], postgres)).to.equal("payload::jsonb #>> '{metadata,filename}'") + + expect(jsonPathNumber('payload', ['duration'], sqlite)).to.equal("json_extract(payload, '$.duration')") + expect(jsonPathNumber('payload', ['duration'], postgres)).to.equal("NULLIF(payload::jsonb #>> '{duration}', '')::double precision") + }) + + it('should generate safe numeric cast expressions for sequence-like text', () => { + expect(safeTextToDoubleExpression('sequence', sqlite)).to.equal('CAST(sequence AS FLOAT)') + expect(safeTextToDoubleExpression('"bookSeries"."sequence"', postgres)).to.equal("CASE WHEN BTRIM(\"bookSeries\".\"sequence\") ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(\"bookSeries\".\"sequence\")::double precision ELSE NULL END") + + expect(safeTextToIntegerExpression('publishedYear', sqlite)).to.equal('CAST(publishedYear AS INTEGER)') + expect(safeTextToIntegerExpression('book.publishedYear', postgres)).to.equal("CASE WHEN BTRIM(book.publishedYear) ~ '^[+-]?\\d+$' THEN BTRIM(book.publishedYear)::integer ELSE NULL END") + }) +})