From 055dbb217c7a9ff1002aa634df3ccbdcce843b87 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 16:52:24 -0500 Subject: [PATCH 01/38] add postgres dialect support for homelab migration --- package-lock.json | 167 +++++++++++- package.json | 3 + server/Database.js | 152 ++++++++--- server/Server.js | 25 +- server/controllers/LibraryController.js | 16 +- server/managers/MigrationManager.js | 52 ++-- .../v2.20.0-improve-author-sort-queries.js | 10 + server/models/Author.js | 9 +- server/models/Book.js | 9 +- server/models/LibraryItem.js | 13 +- server/models/Series.js | 9 +- server/scanner/LibraryScanner.js | 7 +- server/scripts/migrateSqliteToPostgres.js | 243 ++++++++++++++++++ server/utils/queries/libraryFilters.js | 7 +- server/utils/queries/libraryItemFilters.js | 11 +- .../utils/queries/libraryItemsBookFilters.js | 93 +++++-- .../queries/libraryItemsPodcastFilters.js | 25 +- server/utils/queries/seriesFilters.js | 21 +- server/utils/sqlDialectHelpers.js | 75 ++++++ test/server/utils/sqlDialectHelpers.test.js | 58 +++++ 20 files changed, 848 insertions(+), 157 deletions(-) create mode 100644 server/scripts/migrateSqliteToPostgres.js create mode 100644 server/utils/sqlDialectHelpers.js create mode 100644 test/server/utils/sqlDialectHelpers.test.js diff --git a/package-lock.json b/package-lock.json index 47d882171..bc23a15c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,8 @@ "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", + "pg": "^8.19.0", + "pg-hstore": "^2.3.4", "semver": "^7.6.3", "sequelize": "^6.35.2", "socket.io": "^4.5.4", @@ -4138,10 +4140,106 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" }, + "node_modules/pg": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", + "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.15.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, "node_modules/pg-connection-string": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", - "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-hstore": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz", + "integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==", + "license": "MIT", + "dependencies": { + "underscore": "^1.13.1" + }, + "engines": { + "node": ">= 0.8.x" + } + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz", + "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } }, "node_modules/picocolors": { "version": "1.0.0", @@ -4173,6 +4271,45 @@ "node": ">=8" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -4946,6 +5083,15 @@ "node": ">=8" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -5265,6 +5411,12 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "node_modules/underscore": { + "version": "1.13.8", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", + "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", + "license": "MIT" + }, "node_modules/unique-filename": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", @@ -5476,6 +5628,15 @@ "node": ">=4.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 534444bb1..b2c76081e 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "docker": "docker buildx build --platform linux/amd64,linux/arm64 --push . -t advplyr/audiobookshelf", "docker-amd64-local": "docker buildx build --platform linux/amd64 --load . -t advplyr/audiobookshelf-amd64-local", "docker-arm64-local": "docker buildx build --platform linux/arm64 --load . -t advplyr/audiobookshelf-arm64-local", + "migrate-sqlite-to-postgres": "node server/scripts/migrateSqliteToPostgres.js", "deploy-linux": "node deploy/linux", "test": "mocha", "coverage": "nyc mocha" @@ -52,6 +53,8 @@ "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", + "pg": "^8.19.0", + "pg-hstore": "^2.3.4", "semver": "^7.6.3", "sequelize": "^6.35.2", "socket.io": "^4.5.4", diff --git a/server/Database.js b/server/Database.js index 213c2c61b..68f6e2729 100644 --- a/server/Database.js +++ b/server/Database.js @@ -14,6 +14,7 @@ class Database { constructor() { this.sequelize = null this.dbPath = null + this.dialect = 'sqlite' this.isNew = false // New absdatabase.sqlite created this.hasRootUser = false // Used to show initialization page in web ui @@ -162,11 +163,37 @@ class Database { return this.models.device } + /** + * @returns {'sqlite'|'postgres'} + */ + getConfiguredDialect() { + const explicitDialect = process.env.DB_DIALECT?.trim()?.toLowerCase() + if (explicitDialect === 'postgres' || explicitDialect === 'sqlite') { + return explicitDialect + } + + const databaseUrl = process.env.DATABASE_URL || '' + if (databaseUrl.startsWith('postgres://') || databaseUrl.startsWith('postgresql://')) { + return 'postgres' + } + + return 'sqlite' + } + + isSqliteDialect() { + return this.dialect === 'sqlite' + } + + isPostgresDialect() { + return this.dialect === 'postgres' + } + /** * Check if db file exists * @returns {boolean} */ async checkHasDb() { + if (!this.isSqliteDialect()) return true if (!(await fs.pathExists(this.dbPath))) { Logger.info(`[Database] absdatabase.sqlite not found at ${this.dbPath}`) return false @@ -174,20 +201,39 @@ class Database { return true } + /** + * Check if any user tables exist (for networked dialects) + * @returns {Promise} + */ + async checkHasTables() { + const queryInterface = this.sequelize.getQueryInterface() + const tables = await queryInterface.showAllTables() + return Array.isArray(tables) && tables.length > 0 + } + /** * Connect to db, build models and run migrations * @param {boolean} [force=false] Used for testing, drops & re-creates all tables */ async init(force = false) { - this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') - - // First check if this is a new database - this.isNew = !(await this.checkHasDb()) || force + this.dialect = this.getConfiguredDialect() + if (this.isSqliteDialect()) { + this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') + // First check if this is a new database + this.isNew = !(await this.checkHasDb()) || force + } else { + this.dbPath = process.env.DATABASE_URL || null + this.isNew = !!force + } if (!(await this.connect())) { throw new Error('Database connection failed') } + if (this.isPostgresDialect() && !force) { + this.isNew = !(await this.checkHasTables()) + } + try { const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath) await migrationManager.init(packageJson.version) @@ -214,7 +260,11 @@ class Database { * @returns {boolean} */ async connect() { - Logger.info(`[Database] Initializing db at "${this.dbPath}"`) + if (this.isSqliteDialect()) { + Logger.info(`[Database] Initializing sqlite db at "${this.dbPath}"`) + } else { + Logger.info(`[Database] Initializing postgres db connection`) + } let logging = false let benchmark = false @@ -229,13 +279,26 @@ class Database { benchmark = true } - this.sequelize = new Sequelize({ - dialect: 'sqlite', - storage: this.dbPath, - logging: logging, - benchmark: benchmark, - transactionType: 'IMMEDIATE' - }) + if (this.isSqliteDialect()) { + this.sequelize = new Sequelize({ + dialect: 'sqlite', + storage: this.dbPath, + logging: logging, + benchmark: benchmark, + transactionType: 'IMMEDIATE' + }) + } else { + if (!process.env.DATABASE_URL) { + Logger.error(`[Database] DATABASE_URL is required when DB_DIALECT=postgres`) + return false + } + + this.sequelize = new Sequelize(process.env.DATABASE_URL, { + dialect: 'postgres', + logging: logging, + benchmark: benchmark + }) + } // Helper function this.sequelize.uppercaseFirst = (str) => (str ? `${str[0].toUpperCase()}${str.substr(1)}` : '') @@ -243,34 +306,36 @@ class Database { try { await this.sequelize.authenticate() - // Set SQLite pragmas from environment variables - const allowedPragmas = [ - { name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' }, - { name: 'cache_size', env: 'SQLITE_CACHE_SIZE' }, - { name: 'temp_store', env: 'SQLITE_TEMP_STORE' } - ] + if (this.isSqliteDialect()) { + // Set SQLite pragmas from environment variables + const allowedPragmas = [ + { name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' }, + { name: 'cache_size', env: 'SQLITE_CACHE_SIZE' }, + { name: 'temp_store', env: 'SQLITE_TEMP_STORE' } + ] - for (const pragma of allowedPragmas) { - const value = process.env[pragma.env] - if (value !== undefined) { - try { - Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`) - await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`) - const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`) - Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result) - } catch (error) { - Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error) + for (const pragma of allowedPragmas) { + const value = process.env[pragma.env] + if (value !== undefined) { + try { + Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`) + await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`) + const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`) + Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result) + } catch (error) { + Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error) + } } } - } - if (process.env.NUSQLITE3_PATH) { - await this.loadExtension(process.env.NUSQLITE3_PATH) - Logger.info(`[Database] Db supports unaccent and unicode foldings`) - this.supportsUnaccent = true - this.supportsUnicodeFoldings = true + if (process.env.NUSQLITE3_PATH) { + await this.loadExtension(process.env.NUSQLITE3_PATH) + Logger.info(`[Database] Db supports unaccent and unicode foldings`) + this.supportsUnaccent = true + this.supportsUnicodeFoldings = true + } } - Logger.info(`[Database] Db connection was successful`) + Logger.info(`[Database] Db connection was successful (${this.dialect})`) return true } catch (error) { Logger.error(`[Database] Failed to connect to db`, error) @@ -307,7 +372,7 @@ class Database { * Disconnect from db */ async disconnect() { - Logger.info(`[Database] Disconnecting sqlite db`) + Logger.info(`[Database] Disconnecting ${this.dialect} db`) await this.sequelize.close() } @@ -315,7 +380,7 @@ class Database { * Reconnect to db and init */ async reconnect() { - Logger.info(`[Database] Reconnecting sqlite db`) + Logger.info(`[Database] Reconnecting ${this.dialect} db`) await this.init() } @@ -368,7 +433,7 @@ class Database { * Loads most of the data from the database. This is a temporary solution. */ async loadData() { - if (this.isNew && (await dbMigration.checkShouldMigrate())) { + if (this.isSqliteDialect() && this.isNew && (await dbMigration.checkShouldMigrate())) { Logger.info(`[Database] New database was created and old database was detected - migrating old to new`) await dbMigration.migrate(this.models) } @@ -847,6 +912,11 @@ WHERE EXISTS ( * It adds triggers to update libraryItems.title[IgnorePrefix] when (books|podcasts).title[IgnorePrefix] is updated */ async addTriggers() { + if (!this.isSqliteDialect()) { + Logger.info(`[Database] Skipping sqlite-only triggers for dialect ${this.dialect}`) + return + } + await this.addTriggerIfNotExists('books', 'title', 'id', 'libraryItems', 'title', 'mediaId') await this.addTriggerIfNotExists('books', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId') await this.addTriggerIfNotExists('podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId') @@ -954,6 +1024,7 @@ WHERE EXISTS ( this.supportsUnaccent = supportsUnaccent this.query = query this.hasAccents = false + this.dialect = sequelize.getDialect() } /** @@ -989,9 +1060,10 @@ WHERE EXISTS ( */ matchExpression(column) { const pattern = this.sequelize.escape(`%${this.query}%`) - if (!this.supportsUnaccent) return `${column} LIKE ${pattern}` + const likeOperator = this.dialect === 'postgres' ? 'ILIKE' : 'LIKE' + if (!this.supportsUnaccent) return `${column} ${likeOperator} ${pattern}` const normalizedColumn = this.hasAccents ? column : this.normalize(column) - return `${normalizedColumn} LIKE ${pattern}` + return `${normalizedColumn} ${likeOperator} ${pattern}` } } } diff --git a/server/Server.js b/server/Server.js index c1657ff1e..b3300344a 100644 --- a/server/Server.js +++ b/server/Server.js @@ -479,14 +479,27 @@ class Server { // Remove series from hide from continue listening that no longer exist try { - const users = await Database.sequelize.query(`SELECT u.id, u.username, u.extraData, json_group_array(value) AS seriesIdsToRemove FROM users u, json_each(u.extraData->"seriesHideFromContinueListening") LEFT JOIN series se ON se.id = value WHERE se.id IS NULL GROUP BY u.id;`, { - model: Database.userModel, - type: Sequelize.QueryTypes.SELECT + const users = await Database.userModel.findAll({ + attributes: ['id', 'username', 'extraData'] }) + for (const user of users) { - const extraData = JSON.parse(user.extraData) - const existingSeriesIds = extraData.seriesHideFromContinueListening - const seriesIdsToRemove = JSON.parse(user.dataValues.seriesIdsToRemove) + const extraData = typeof user.extraData === 'string' ? JSON.parse(user.extraData || '{}') : user.extraData || {} + const existingSeriesIds = Array.isArray(extraData.seriesHideFromContinueListening) ? extraData.seriesHideFromContinueListening : [] + if (!existingSeriesIds.length) continue + + const existingSeries = await Database.seriesModel.findAll({ + attributes: ['id'], + where: { + id: { + [Sequelize.Op.in]: existingSeriesIds + } + } + }) + const existingSeriesSet = new Set(existingSeries.map((series) => series.id)) + const seriesIdsToRemove = existingSeriesIds.filter((seriesId) => !existingSeriesSet.has(seriesId)) + if (!seriesIdsToRemove.length) continue + Logger.info(`[Server] Found ${seriesIdsToRemove.length} non-existent series in seriesHideFromContinueListening for user "${user.username}" - Removing (${seriesIdsToRemove.join(',')})`) const newExtraData = { ...extraData, diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 71f210106..7c7bab873 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -24,6 +24,7 @@ const libraryFilters = require('../utils/queries/libraryFilters') const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters') const authorFilters = require('../utils/queries/authorFilters') const zipHelpers = require('../utils/zipHelpers') +const { isPostgres, noCaseSortExpression, jsonArrayExpand } = require('../utils/sqlDialectHelpers') /** * @typedef RequestUserObject @@ -1039,9 +1040,9 @@ class LibraryController { let order = undefined const direction = payload.sortDesc ? 'DESC' : 'ASC' if (payload.sortBy === 'name') { - order = [[Sequelize.literal('name COLLATE NOCASE'), direction]] + order = [[Sequelize.literal(noCaseSortExpression('name', Database.sequelize)), direction]] } else if (payload.sortBy === 'lastFirst') { - order = [[Sequelize.literal('lastFirst COLLATE NOCASE'), direction]] + order = [[Sequelize.literal(noCaseSortExpression('lastFirst', Database.sequelize)), direction]] } else if (payload.sortBy === 'addedAt') { order = [['createdAt', direction]] } else if (payload.sortBy === 'updatedAt') { @@ -1338,16 +1339,23 @@ class LibraryController { const fileExt = req.query.ext === 'abs' ? 'abs' : 'json' const metadataFilename = `metadata.${fileExt}` + const metadataFilenameCountQuery = isPostgres(Database.sequelize) + ? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{metadata,filename}' = :metadataFilename)` + : `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = :metadataFilename)` + const libraryItemsWithMetadata = await Database.libraryItemModel.findAll({ attributes: ['id', 'libraryFiles'], where: [ { libraryId: req.library.id }, - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = "${metadataFilename}")`), { + Sequelize.where(Sequelize.literal(metadataFilenameCountQuery), { [Sequelize.Op.gte]: 1 }) - ] + ], + replacements: { + metadataFilename + } }) if (!libraryItemsWithMetadata.length) { Logger.info(`[LibraryController] No ${metadataFilename} files found to remove`) diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index e302038ff..588771318 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -94,20 +94,26 @@ class MigrationManager { // Only proceed with migration if there are migrations to run if (migrationsToRun.length > 0) { + const dialect = typeof this.sequelize.getDialect === 'function' ? this.sequelize.getDialect() : 'sqlite' + const isSqlite = !dialect || dialect === 'sqlite' const originalDbPath = path.join(this.configPath, 'absdatabase.sqlite') const backupDbPath = path.join(this.configPath, 'absdatabase.backup.sqlite') try { Logger.info(`[MigrationManager] Migrating database ${migrationDirection} to version ${this.serverVersion}`) Logger.info(`[MigrationManager] Migrations to run: ${migrationsToRun.join(', ')}`) - // Create a backup copy of the SQLite database before starting migrations - await fs.copy(originalDbPath, backupDbPath) - Logger.info('Created a backup of the original database.') + if (isSqlite) { + // Create a backup copy of the SQLite database before starting migrations + await fs.copy(originalDbPath, backupDbPath) + Logger.info('Created a backup of the original database.') + } // Run migrations await this.umzug[migrationDirection]({ migrations: migrationsToRun, rerun: 'ALLOW' }) - // Clean up the backup - await fs.remove(backupDbPath) + if (isSqlite) { + // Clean up the backup + await fs.remove(backupDbPath) + } Logger.info('[MigrationManager] Migrations successfully applied to the original database.') } catch (error) { @@ -115,13 +121,15 @@ class MigrationManager { await this.sequelize.close() - // Step 3: If migration fails, save the failed original and restore the backup - const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite') - await fs.move(originalDbPath, failedDbPath, { overwrite: true }) - Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.') + if (isSqlite) { + // Step 3: If migration fails, save the failed original and restore the backup + const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite') + await fs.move(originalDbPath, failedDbPath, { overwrite: true }) + Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.') - await fs.move(backupDbPath, originalDbPath, { overwrite: true }) - Logger.info('[MigrationManager] Restored the original database from the backup.') + await fs.move(backupDbPath, originalDbPath, { overwrite: true }) + Logger.info('[MigrationManager] Restored the original database from the backup.') + } Logger.info('[MigrationManager] Migration failed. Exiting Audiobookshelf with code 1.') process.exit(1) @@ -191,16 +199,15 @@ class MigrationManager { } async fetchVersionsFromDatabase() { + const migrationsMetaTable = `"${MigrationManager.MIGRATIONS_META_TABLE}"` await this.checkOrCreateMigrationsMetaTable() - const [{ version }] = await this.sequelize.query("SELECT value as version FROM :migrationsMeta WHERE key = 'version'", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [{ version }] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, { type: Sequelize.QueryTypes.SELECT }) this.databaseVersion = version - const [{ maxVersion }] = await this.sequelize.query("SELECT value as maxVersion FROM :migrationsMeta WHERE key = 'maxVersion'", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [{ maxVersion }] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, { type: Sequelize.QueryTypes.SELECT }) this.maxVersion = maxVersion @@ -212,8 +219,7 @@ class MigrationManager { // If the table exists, check that the `version` and `maxVersion` rows exist if (migrationsMetaTableExists) { - const [{ count }] = await this.sequelize.query("SELECT COUNT(*) as count FROM :migrationsMeta WHERE key IN ('version', 'maxVersion')", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM "${MigrationManager.MIGRATIONS_META_TABLE}" WHERE key IN ('version', 'maxVersion')`, { type: Sequelize.QueryTypes.SELECT }) if (count < 2) { @@ -241,8 +247,8 @@ class MigrationManager { allowNull: false } }) - await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", { - replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`INSERT INTO "${MigrationManager.MIGRATIONS_META_TABLE}" (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, { + replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0' }, type: Sequelize.QueryTypes.INSERT }) Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`) @@ -299,8 +305,8 @@ class MigrationManager { async updateMaxVersion() { try { - await this.sequelize.query("UPDATE :migrationsMeta SET value = :maxVersion WHERE key = 'maxVersion'", { - replacements: { maxVersion: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :maxVersion WHERE key = 'maxVersion'`, { + replacements: { maxVersion: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) } catch (error) { @@ -311,8 +317,8 @@ class MigrationManager { async updateDatabaseVersion() { try { - await this.sequelize.query("UPDATE :migrationsMeta SET value = :version WHERE key = 'version'", { - replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :version WHERE key = 'version'`, { + replacements: { version: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) } catch (error) { diff --git a/server/migrations/v2.20.0-improve-author-sort-queries.js b/server/migrations/v2.20.0-improve-author-sort-queries.js index 53016c709..c5c2b8f5a 100644 --- a/server/migrations/v2.20.0-improve-author-sort-queries.js +++ b/server/migrations/v2.20.0-improve-author-sort-queries.js @@ -39,6 +39,11 @@ const authorsJoin = `${authors} JOIN ${bookAuthors} ON ${authors}.id = ${bookAut * @returns {Promise} - A promise that resolves when the migration is complete. */ async function up({ context: { queryInterface, logger } }) { + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`) + return + } + const helper = new MigrationHelper(queryInterface, logger) // Upwards migration script @@ -72,6 +77,11 @@ async function up({ context: { queryInterface, logger } }) { * @returns {Promise} - A promise that resolves when the migration is complete. */ async function down({ context: { queryInterface, logger } }) { + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`) + return + } + // Downward migration script logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) diff --git a/server/models/Author.js b/server/models/Author.js index 65561e211..f9a670351 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -129,6 +129,8 @@ class Author extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' } + super.init( { id: { @@ -147,12 +149,7 @@ class Author extends Model { modelName: 'author', indexes: [ { - fields: [ - { - name: 'name', - collate: 'NOCASE' - } - ] + fields: [nameIndexField] }, // { // fields: [{ diff --git a/server/models/Book.js b/server/models/Book.js index 7e1a2e30c..d8ce6ff29 100644 --- a/server/models/Book.js +++ b/server/models/Book.js @@ -138,6 +138,8 @@ class Book extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' } + super.init( { id: { @@ -172,12 +174,7 @@ class Book extends Model { modelName: 'book', indexes: [ { - fields: [ - { - name: 'title', - collate: 'NOCASE' - } - ] + fields: [titleIndexField] }, // { // fields: [{ diff --git a/server/models/LibraryItem.js b/server/models/LibraryItem.js index c03667efc..6c464502e 100644 --- a/server/models/LibraryItem.js +++ b/server/models/LibraryItem.js @@ -720,6 +720,11 @@ class LibraryItem extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' } + const titleIgnorePrefixIndexField = sequelize.getDialect() === 'postgres' ? 'titleIgnorePrefix' : { name: 'titleIgnorePrefix', collate: 'NOCASE' } + const authorNamesFirstLastIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesFirstLast' : { name: 'authorNamesFirstLast', collate: 'NOCASE' } + const authorNamesLastFirstIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesLastFirst' : { name: 'authorNamesLastFirst', collate: 'NOCASE' } + super.init( { id: { @@ -768,16 +773,16 @@ class LibraryItem extends Model { fields: ['libraryId', 'mediaType', 'createdAt'] }, { - fields: ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', titleIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', titleIgnorePrefixIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', authorNamesFirstLastIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', authorNamesLastFirstIndexField] }, { fields: ['libraryId', 'mediaId', 'mediaType'] diff --git a/server/models/Series.js b/server/models/Series.js index 6ca288464..28450a010 100644 --- a/server/models/Series.js +++ b/server/models/Series.js @@ -87,6 +87,8 @@ class Series extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' } + super.init( { id: { @@ -103,12 +105,7 @@ class Series extends Model { modelName: 'series', indexes: [ { - fields: [ - { - name: 'name', - collate: 'NOCASE' - } - ] + fields: [nameIndexField] }, // { // fields: [{ diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 73cb51553..a34750f07 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -14,6 +14,7 @@ const LibraryItemScanner = require('./LibraryItemScanner') const LibraryScan = require('./LibraryScan') const LibraryItemScanData = require('./LibraryItemScanData') const Task = require('../objects/Task') +const { isPostgres, jsonArrayExpand } = require('../utils/sqlDialectHelpers') class LibraryScanner { constructor() { @@ -676,12 +677,16 @@ async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingle // check if it was moved from another folder by comparing the ino to the library files const ino = await fileUtils.getIno(fullPath) if (!ino) return null + const inodeMatchQuery = isPostgres(Database.sequelize) + ? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{ino}' = :inode)` + : `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)` + const existingLibraryItem = await Database.libraryItemModel.findOneExpanded( [ { libraryId: libraryId }, - sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), { + sequelize.where(sequelize.literal(inodeMatchQuery), { [sequelize.Op.gt]: 0 }) ], diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js new file mode 100644 index 000000000..60acc11e2 --- /dev/null +++ b/server/scripts/migrateSqliteToPostgres.js @@ -0,0 +1,243 @@ +#!/usr/bin/env node + +const sqlite3 = require('sqlite3') +const { Client } = require('pg') + +const SQLITE_PATH = process.env.SQLITE_PATH || '/config/absdatabase.sqlite' +const DATABASE_URL = process.env.DATABASE_URL +const PG_SCHEMA = process.env.PG_SCHEMA || 'public' +const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500) +const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true' + +if (!DATABASE_URL) { + console.error('[migrate] DATABASE_URL is required') + process.exit(1) +} + +const preferredOrder = [ + 'migrationsMeta', + 'SequelizeMeta', + 'settings', + 'users', + 'apiKeys', + 'sessions', + 'libraries', + 'libraryFolders', + 'authors', + 'series', + 'books', + 'podcasts', + 'podcastEpisodes', + 'libraryItems', + 'bookAuthors', + 'bookSeries', + 'collections', + 'collectionBooks', + 'playlists', + 'playlistMediaItems', + 'mediaProgresses', + 'devices', + 'playbackSessions', + 'feeds', + 'feedEpisodes', + 'mediaItemShares', + 'customMetadataProviders' +] + +function quoteIdent(identifier) { + return `"${String(identifier).replace(/"/g, '""')}"` +} + +function openSqlite(filePath) { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(filePath, sqlite3.OPEN_READONLY, (err) => { + if (err) return reject(err) + resolve(db) + }) + }) +} + +function sqliteAll(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => { + if (err) return reject(err) + resolve(rows) + }) + }) +} + +function sqliteGet(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.get(sql, params, (err, row) => { + if (err) return reject(err) + resolve(row) + }) + }) +} + +function normalizeBoolean(value) { + if (value === null || value === undefined) return null + if (typeof value === 'boolean') return value + if (typeof value === 'number') return value !== 0 + if (typeof value === 'string') { + const v = value.trim().toLowerCase() + return v === 'true' || v === '1' || v === 't' + } + return !!value +} + +function normalizeJson(value) { + if (value === null || value === undefined || value === '') return null + if (typeof value === 'object') return value + try { + return JSON.parse(value) + } catch (error) { + return value + } +} + +function convertValue(value, pgColumn) { + if (!pgColumn) return value + const dataType = pgColumn.data_type + const udtName = pgColumn.udt_name + + if (dataType === 'boolean') { + return normalizeBoolean(value) + } + + if (dataType === 'json' || dataType === 'jsonb' || udtName === 'json' || udtName === 'jsonb') { + return normalizeJson(value) + } + + return value +} + +async function main() { + console.log(`[migrate] sqlite source: ${SQLITE_PATH}`) + console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`) + console.log(`[migrate] dry run: ${DRY_RUN}`) + + const sqliteDb = await openSqlite(SQLITE_PATH) + const pg = new Client({ connectionString: DATABASE_URL }) + await pg.connect() + + try { + const sqliteTablesRows = await sqliteAll( + sqliteDb, + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + ) + const sqliteTables = sqliteTablesRows.map((row) => row.name) + + const pgTablesRows = await pg.query( + `SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`, + [PG_SCHEMA] + ) + const pgTables = new Set(pgTablesRows.rows.map((row) => row.table_name)) + + const tablesToMigrate = sqliteTables.filter((table) => pgTables.has(table)) + tablesToMigrate.sort((a, b) => { + const ai = preferredOrder.indexOf(a) + const bi = preferredOrder.indexOf(b) + if (ai === -1 && bi === -1) return a.localeCompare(b) + if (ai === -1) return 1 + if (bi === -1) return -1 + return ai - bi + }) + + if (!tablesToMigrate.length) { + throw new Error('No overlapping tables found between SQLite and PostgreSQL') + } + + console.log(`[migrate] tables to migrate: ${tablesToMigrate.join(', ')}`) + + const pgColumnsByTable = new Map() + for (const table of tablesToMigrate) { + const columnsResult = await pg.query( + `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, + [PG_SCHEMA, table] + ) + const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name, column])) + pgColumnsByTable.set(table, columnMap) + } + + if (!DRY_RUN) { + await pg.query('BEGIN') + await pg.query('SET session_replication_role = replica') + + const truncateList = tablesToMigrate.map((table) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`).join(', ') + await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`) + console.log('[migrate] truncated target tables') + } + + for (const table of tablesToMigrate) { + const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(table)}`) + const pgColumns = pgColumnsByTable.get(table) + const insertColumns = rows.length ? Object.keys(rows[0]).filter((column) => pgColumns.has(column)) : [] + + if (!rows.length || !insertColumns.length) { + console.log(`[migrate] ${table}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`) + continue + } + + if (!DRY_RUN) { + for (let offset = 0; offset < rows.length; offset += BATCH_SIZE) { + const batchRows = rows.slice(offset, offset + BATCH_SIZE) + const valuesSql = [] + const params = [] + let paramIndex = 1 + + for (const row of batchRows) { + const placeholders = [] + for (const column of insertColumns) { + const pgColumn = pgColumns.get(column) + params.push(convertValue(row[column], pgColumn)) + placeholders.push(`$${paramIndex++}`) + } + valuesSql.push(`(${placeholders.join(', ')})`) + } + + const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)} (${insertColumns.map(quoteIdent).join(', ')}) VALUES ${valuesSql.join(', ')}` + await pg.query(insertSql, params) + } + } + + console.log(`[migrate] ${table}: ${rows.length} rows`) + } + + if (!DRY_RUN) { + await pg.query('SET session_replication_role = DEFAULT') + await pg.query('COMMIT') + console.log('[migrate] migration transaction committed') + } + + const parity = [] + for (const table of tablesToMigrate) { + const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(table)}`) + const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`) + parity.push({ + table, + sqliteCount: Number(sqliteCountRow.count || 0), + postgresCount: Number(pgCountResult.rows[0].count || 0) + }) + } + + const mismatches = parity.filter((row) => row.sqliteCount !== row.postgresCount) + if (mismatches.length) { + console.error('[migrate] row-count mismatches detected:') + mismatches.forEach((row) => { + console.error(`[migrate] ${row.table}: sqlite=${row.sqliteCount} postgres=${row.postgresCount}`) + }) + process.exitCode = 2 + } else { + console.log('[migrate] parity check passed for all migrated tables') + } + } finally { + sqliteDb.close() + await pg.end() + } +} + +main().catch((error) => { + console.error('[migrate] failed:', error) + process.exit(1) +}) diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index 7312b9d5d..49f5381e1 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -5,6 +5,7 @@ const libraryItemsBookFilters = require('./libraryItemsBookFilters') const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters') const { createNewSortInstance } = require('../../libs/fastSort') const { profile } = require('../../utils/profiler') +const { booleanLiteral, jsonArrayContainsAny } = require('../sqlDialectHelpers') const naturalSort = createNewSortInstance({ comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare }) @@ -235,13 +236,13 @@ module.exports = { if (userPermissionBookWhere.bookWhere.length) { let attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id' if (!user.canAccessExplicitContent) { - attrQuery += ' AND b.explicit = 0' + attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}` } if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (user.permissions.selectedTagsNotAccessible) { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0` } else { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0` } } seriesWhere.push( diff --git a/server/utils/queries/libraryItemFilters.js b/server/utils/queries/libraryItemFilters.js index 7f95d0ecc..15c3dedb7 100644 --- a/server/utils/queries/libraryItemFilters.js +++ b/server/utils/queries/libraryItemFilters.js @@ -2,6 +2,7 @@ const Sequelize = require('sequelize') const Database = require('../../Database') const libraryItemsBookFilters = require('./libraryItemsBookFilters') const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters') +const { jsonArrayContainsAny } = require('../sqlDialectHelpers') module.exports = { /** @@ -12,7 +13,7 @@ module.exports = { async getAllLibraryItemsWithTags(tags) { const libraryItems = [] const booksWithTag = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -46,7 +47,7 @@ module.exports = { libraryItems.push(libraryItem) } const podcastsWithTag = await Database.podcastModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -77,7 +78,7 @@ module.exports = { async getAllLibraryItemsWithGenres(genres) { const libraryItems = [] const booksWithGenre = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -107,7 +108,7 @@ module.exports = { libraryItems.push(libraryItem) } const podcastsWithGenre = await Database.podcastModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -138,7 +139,7 @@ module.exports = { async getAllLibraryItemsWithNarrators(narrators) { const libraryItems = [] const booksWithGenre = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(narrators) WHERE json_valid(narrators) AND json_each.value IN (:narrators))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('narrators', 'narrators', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index fbe0c4f0d..31df0d735 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -6,6 +6,7 @@ const authorFilters = require('./authorFilters') const ShareManager = require('../../managers/ShareManager') const { profile } = require('../profiler') const stringifySequelizeQuery = require('../stringifySequelizeQuery') +const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand } = require('../sqlDialectHelpers') const countCache = new Map() module.exports = { @@ -27,10 +28,10 @@ module.exports = { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { replacements['userTagsSelected'] = user.permissions.itemTagsSelected if (user.permissions.selectedTagsNotAccessible) { - bookWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0)) + bookWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0)) } else { bookWhere.push( - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), { + Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) ) @@ -189,7 +190,7 @@ module.exports = { } else if (group === 'explicit') { mediaWhere['explicit'] = true } else if (['genres', 'tags', 'narrators'].includes(group)) { - mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), { + mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) replacements.filterValue = value @@ -256,9 +257,9 @@ module.exports = { const getTitleOrder = () => { if (global.ServerSettings.sortingIgnorePrefix) { - return [Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir] + return [Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir] } else { - return [Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir] + return [Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir] } } @@ -273,20 +274,23 @@ module.exports = { } else if (sortBy === 'media.duration') { return [['duration', dir]] } else if (sortBy === 'media.metadata.publishedYear') { - return [[Sequelize.literal(`CAST(\`book\`.\`publishedYear\` AS INTEGER)`), dir]] + return [[Sequelize.literal('CAST(book.publishedYear AS INTEGER)'), dir]] } else if (sortBy === 'media.metadata.authorNameLF') { // Sort by author name last first, secondary sort by title - return [[Sequelize.literal('`libraryItem`.`authorNamesLastFirst` COLLATE NOCASE'), dir], getTitleOrder()] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesLastFirst', Database.sequelize)), dir], getTitleOrder()] } else if (sortBy === 'media.metadata.authorName') { // Sort by author name first last, secondary sort by title - return [[Sequelize.literal('`libraryItem`.`authorNamesFirstLast` COLLATE NOCASE'), dir], getTitleOrder()] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesFirstLast', Database.sequelize)), dir], getTitleOrder()] } else if (sortBy === 'media.metadata.title') { if (collapseseries) { - return [[Sequelize.literal('display_title COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('display_title', Database.sequelize)), dir]] } return [getTitleOrder()] } else if (sortBy === 'sequence') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' + if (Database.sequelize.getDialect() === 'postgres') { + return [[Sequelize.literal(`CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ${nullDir}`)]] + } return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]] } else if (sortBy === 'progress') { return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]] @@ -326,7 +330,13 @@ module.exports = { required: true } ], - order: [Sequelize.literal('CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')] + order: [ + Sequelize.literal( + Database.sequelize.getDialect() === 'postgres' + ? 'CAST("books.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST' + : 'CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST' + ) + ] }) const bookSeriesToInclude = [] const booksToInclude = [] @@ -455,12 +465,28 @@ module.exports = { }) } else if (filterGroup === 'ebooks' && filterValue === 'supplementary') { // TODO: Temp workaround for filtering supplementary ebook - libraryItemWhere['libraryFiles'] = { - [Sequelize.Op.substring]: `"isSupplementary":true` + if (Database.sequelize.getDialect() === 'postgres') { + libraryItemWhere[Sequelize.Op.and] = [ + Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), { + [Sequelize.Op.like]: '%"isSupplementary":true%' + }) + ] + } else { + libraryItemWhere['libraryFiles'] = { + [Sequelize.Op.substring]: `"isSupplementary":true` + } } } else if (filterGroup === 'ebooks' && filterValue === 'no-supplementary') { - libraryItemWhere['libraryFiles'] = { - [Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`) + if (Database.sequelize.getDialect() === 'postgres') { + libraryItemWhere[Sequelize.Op.and] = [ + Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), { + [Sequelize.Op.notLike]: '%"isSupplementary":true%' + }) + ] + } else { + libraryItemWhere['libraryFiles'] = { + [Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`) + } } } else if (filterGroup === 'missing' && filterValue === 'authors') { authorInclude = { @@ -502,7 +528,13 @@ module.exports = { }) if (sortBy !== 'sequence') { // Secondary sort by sequence - sortOrder.push([Sequelize.literal('CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]) + sortOrder.push([ + Sequelize.literal( + Database.sequelize.getDialect() === 'postgres' + ? 'CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST' + : 'CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST' + ) + ]) } } else if (filterGroup === 'issues') { libraryItemWhere[Sequelize.Op.or] = [ @@ -594,10 +626,14 @@ module.exports = { // When collapsing series and sorting by title then use the series name instead of the book title // for this set an attribute "display_title" to use in sorting + const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."title"' : '`libraryItem`.`title`' + const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."titleIgnorePrefix"' : '`libraryItem`.`titleIgnorePrefix`' + const fallbackFn = coalesceFunctionName(Database.sequelize) + if (global.ServerSettings.sortingIgnorePrefix) { - bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`titleIgnorePrefix\`)`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) } else { - bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`title\`)`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitle})`), 'display_title']) } } @@ -716,13 +752,14 @@ module.exports = { bookWhere.push(...userPermissionBookWhere.bookWhere) let includeAttributes = [[Sequelize.literal('(SELECT max(mp.updatedAt) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id)'), 'recent_progress']] - let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = 0 OR mp.isFinished IS NULL)` + let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = ${booleanLiteral(false, Database.sequelize)} OR mp.isFinished IS NULL)` if (library.settings.onlyShowLaterBooksInContinueSeries) { - const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as FLOAT) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = 1 AND mp.userId = :userId AND bs.seriesId = series.id)` + const castType = Database.sequelize.getDialect() === 'postgres' ? 'DOUBLE PRECISION' : 'FLOAT' + const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as ${castType}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)` includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence']) - booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as FLOAT) > ${maxSequenceQuery}` + booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as ${castType}) > ${maxSequenceQuery}` } const { rows: series, count } = await Database.seriesModel.findAndCountAll({ @@ -735,7 +772,7 @@ module.exports = { }, // TODO: Simplify queries // Has at least 1 book finished - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = 1)`), { + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)})`), { [Sequelize.Op.gte]: 1 }), // Has at least 1 book not finished (that has a sequence number higher than the highest already read, if library config is toggled) @@ -743,7 +780,7 @@ module.exports = { [Sequelize.Op.gte]: 1 }), // Has no books in progress - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = 0 AND mp.currentTime > 0)`), 0) + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)} AND mp.currentTime > 0)`), 0) ], attributes: { include: includeAttributes @@ -760,7 +797,7 @@ module.exports = { order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], where: { '$book.mediaProgresses.isFinished$': { - [Sequelize.Op.or]: [null, 0] + [Sequelize.Op.or]: [null, false] } }, include: { @@ -854,7 +891,7 @@ module.exports = { { libraryId }, - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = 1 OR mp.currentTime > 0))`), 0) + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0))`), 0) ], replacements: { userId: user.id, @@ -1174,7 +1211,7 @@ module.exports = { // Search narrators const narratorMatches = [] - const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, { + const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, ${jsonArrayExpand('b.narrators', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1191,7 +1228,7 @@ module.exports = { // Search tags const tagMatches = [] - const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.tags', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1208,7 +1245,7 @@ module.exports = { // Search genres const genreMatches = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1288,7 +1325,7 @@ module.exports = { */ async getGenresWithCount(libraryId) { const genres = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { replacements: { libraryId }, diff --git a/server/utils/queries/libraryItemsPodcastFilters.js b/server/utils/queries/libraryItemsPodcastFilters.js index 8bb5dc110..d89a4bd73 100644 --- a/server/utils/queries/libraryItemsPodcastFilters.js +++ b/server/utils/queries/libraryItemsPodcastFilters.js @@ -3,6 +3,7 @@ const Database = require('../../Database') const Logger = require('../../Logger') const { profile } = require('../../utils/profiler') const stringifySequelizeQuery = require('../stringifySequelizeQuery') +const { jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression, jsonArrayExpand, jsonPathNumber } = require('../sqlDialectHelpers') const countCache = new Map() @@ -24,10 +25,10 @@ module.exports = { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { replacements['userTagsSelected'] = user.permissions.itemTagsSelected if (user.permissions.selectedTagsNotAccessible) { - podcastWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0)) + podcastWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0)) } else { podcastWhere.push( - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), { + Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) ) @@ -53,7 +54,7 @@ module.exports = { const replacements = {} if (['genres', 'tags'].includes(group)) { - mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), { + mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) replacements.filterValue = value @@ -87,12 +88,12 @@ module.exports = { return [[Sequelize.literal('libraryItem.mtime'), dir]] } else if (sortBy === 'media.metadata.author') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' - return [[Sequelize.literal(`\`podcast\`.\`author\` COLLATE NOCASE ${nullDir}`)]] + return [[Sequelize.literal(`${noCaseSortExpression('podcast.author', Database.sequelize)} ${nullDir}`)]] } else if (sortBy === 'media.metadata.title') { if (global.ServerSettings.sortingIgnorePrefix) { - return [[Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]] } else { - return [[Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]] } } else if (sortBy === 'media.numTracks') { return [['numEpisodes', dir]] @@ -455,7 +456,7 @@ module.exports = { // Search tags const tagMatches = [] - const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.tags', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -472,7 +473,7 @@ module.exports = { // Search genres const genreMatches = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -563,12 +564,12 @@ module.exports = { * @returns {Promise<{ totalSize:number, totalDuration:number, numAudioFiles:number, totalItems:number}>} */ async getPodcastLibraryStats(libraryId) { - const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = "podcast" AND li.libraryId = :libraryId;`, { + const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = 'podcast' AND li.libraryId = :libraryId;`, { replacements: { libraryId } }) - const [statResults] = await Database.sequelize.query(`SELECT SUM(json_extract(pe.audioFile, '$.duration')) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, { + const [statResults] = await Database.sequelize.query(`SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, { replacements: { libraryId } @@ -588,7 +589,7 @@ module.exports = { */ async getGenresWithCount(libraryId) { const genres = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { replacements: { libraryId }, @@ -611,7 +612,7 @@ module.exports = { */ async getLongestPodcasts(libraryId, limit) { const podcasts = await Database.podcastModel.findAll({ - attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(json_extract(pe.audioFile, '$.duration')) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']], + attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']], include: { model: Database.libraryItemModel, attributes: ['id', 'libraryId'], diff --git a/server/utils/queries/seriesFilters.js b/server/utils/queries/seriesFilters.js index ed71e5b3f..3aa208552 100644 --- a/server/utils/queries/seriesFilters.js +++ b/server/utils/queries/seriesFilters.js @@ -2,6 +2,7 @@ const Sequelize = require('sequelize') const Logger = require('../../Logger') const Database = require('../../Database') const libraryItemsBookFilters = require('./libraryItemsBookFilters') +const { booleanLiteral, jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression } = require('../sqlDialectHelpers') module.exports = { decode(text) { @@ -60,7 +61,7 @@ module.exports = { // TODO: Simplify and break-out let attrQuery = null if (['genres', 'tags', 'narrators'].includes(filterGroup)) { - attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (SELECT count(*) FROM json_each(b.${filterGroup}) WHERE json_valid(b.${filterGroup}) AND json_each.value = :filterValue) > 0` + attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ${jsonArrayContainsValue(`b.${filterGroup}`, 'filterValue', Database.sequelize)} > 0` userPermissionBookWhere.replacements.filterValue = filterValue } else if (filterGroup === 'authors') { attrQuery = 'SELECT count(*) FROM books b, bookSeries bs, bookAuthors ba WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ba.bookId = b.id AND ba.authorId = :filterValue' @@ -73,18 +74,18 @@ module.exports = { userPermissionBookWhere.replacements.filterValue = filterValue } else if (filterGroup === 'progress') { if (filterValue === 'not-finished') { - attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)' + attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})` userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'finished') { - const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)' + const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})` seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0)) userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'not-started') { - const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = 1 OR mp.currentTime > 0)' + const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0)` seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0)) userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'in-progress') { - attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = 0' + attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)}` userPermissionBookWhere.replacements.userId = user.id } } @@ -95,13 +96,13 @@ module.exports = { if (!attrQuery) attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id' if (!user.canAccessExplicitContent) { - attrQuery += ' AND b.explicit = 0' + attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}` } if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (user.permissions.selectedTagsNotAccessible) { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0` } else { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0` } } } @@ -128,9 +129,9 @@ module.exports = { order.push(['createdAt', dir]) } else if (sortBy === 'name') { if (global.ServerSettings.sortingIgnorePrefix) { - order.push([Sequelize.literal('nameIgnorePrefix COLLATE NOCASE'), dir]) + order.push([Sequelize.literal(noCaseSortExpression('nameIgnorePrefix', Database.sequelize)), dir]) } else { - order.push([Sequelize.literal('`series`.`name` COLLATE NOCASE'), dir]) + order.push([Sequelize.literal(noCaseSortExpression('series.name', Database.sequelize)), dir]) } } else if (sortBy === 'totalDuration') { seriesAttributes.include.push([Sequelize.literal('(SELECT SUM(b.duration) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND b.id = bs.bookId)'), 'totalDuration']) diff --git a/server/utils/sqlDialectHelpers.js b/server/utils/sqlDialectHelpers.js new file mode 100644 index 000000000..36538bcb1 --- /dev/null +++ b/server/utils/sqlDialectHelpers.js @@ -0,0 +1,75 @@ +function getDialect(sequelize) { + if (!sequelize) return 'sqlite' + if (typeof sequelize.getDialect === 'function') return sequelize.getDialect() + return sequelize.dialect?.name || 'sqlite' +} + +function isPostgres(sequelize) { + return getDialect(sequelize) === 'postgres' +} + +function booleanLiteral(value, sequelize) { + if (isPostgres(sequelize)) return value ? 'TRUE' : 'FALSE' + return value ? '1' : '0' +} + +function noCaseSortExpression(columnExpression, sequelize) { + if (isPostgres(sequelize)) return `LOWER(${columnExpression})` + return `${columnExpression} COLLATE NOCASE` +} + +function coalesceFunctionName(sequelize) { + return isPostgres(sequelize) ? 'COALESCE' : 'IFNULL' +} + +function jsonArrayContainsAny(columnExpression, bindName, sequelize) { + if (isPostgres(sequelize)) { + return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:${bindName}))` + } + return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value IN (:${bindName}))` +} + +function jsonArrayContainsValue(columnExpression, bindName, sequelize) { + if (isPostgres(sequelize)) { + return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :${bindName})` + } + return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value = :${bindName})` +} + +function jsonArrayExpand(columnExpression, sequelize, options = {}) { + const alias = options.alias || 'json_each' + const textValues = options.textValues !== false + if (isPostgres(sequelize)) { + const fn = textValues ? 'jsonb_array_elements_text' : 'jsonb_array_elements' + return `${fn}(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS ${alias}(value)` + } + return `json_each(${columnExpression})` +} + +function jsonPathText(columnExpression, pathSegments, sequelize) { + const path = Array.isArray(pathSegments) ? pathSegments : [pathSegments] + if (isPostgres(sequelize)) { + return `${columnExpression}::jsonb #>> '{${path.join(',')}}'` + } + return `json_extract(${columnExpression}, '$.${path.join('.')}')` +} + +function jsonPathNumber(columnExpression, pathSegments, sequelize) { + if (isPostgres(sequelize)) { + return `NULLIF(${jsonPathText(columnExpression, pathSegments, sequelize)}, '')::double precision` + } + return `json_extract(${columnExpression}, '$.${[].concat(pathSegments).join('.')}')` +} + +module.exports = { + getDialect, + isPostgres, + booleanLiteral, + noCaseSortExpression, + coalesceFunctionName, + jsonArrayContainsAny, + jsonArrayContainsValue, + jsonArrayExpand, + jsonPathText, + jsonPathNumber +} diff --git a/test/server/utils/sqlDialectHelpers.test.js b/test/server/utils/sqlDialectHelpers.test.js new file mode 100644 index 000000000..006b02364 --- /dev/null +++ b/test/server/utils/sqlDialectHelpers.test.js @@ -0,0 +1,58 @@ +const { expect } = require('chai') + +const { + booleanLiteral, + noCaseSortExpression, + coalesceFunctionName, + jsonArrayContainsAny, + jsonArrayContainsValue, + jsonArrayExpand, + jsonPathText, + jsonPathNumber +} = require('../../../server/utils/sqlDialectHelpers') + +const sqlite = { + getDialect: () => 'sqlite' +} + +const postgres = { + getDialect: () => 'postgres' +} + +describe('sqlDialectHelpers', () => { + it('should return sqlite and postgres boolean literals', () => { + expect(booleanLiteral(true, sqlite)).to.equal('1') + expect(booleanLiteral(false, sqlite)).to.equal('0') + expect(booleanLiteral(true, postgres)).to.equal('TRUE') + expect(booleanLiteral(false, postgres)).to.equal('FALSE') + }) + + it('should generate case-insensitive sort expressions', () => { + expect(noCaseSortExpression('name', sqlite)).to.equal('name COLLATE NOCASE') + expect(noCaseSortExpression('name', postgres)).to.equal('LOWER(name)') + }) + + it('should choose the correct null-coalescing function name', () => { + expect(coalesceFunctionName(sqlite)).to.equal('IFNULL') + expect(coalesceFunctionName(postgres)).to.equal('COALESCE') + }) + + it('should generate array membership count queries per dialect', () => { + expect(jsonArrayContainsAny('tags', 'selectedTags', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:selectedTags))') + expect(jsonArrayContainsAny('tags', 'selectedTags', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:selectedTags))") + + expect(jsonArrayContainsValue('tags', 'tag', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value = :tag)') + expect(jsonArrayContainsValue('tags', 'tag', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :tag)") + }) + + it('should generate json array expansion and path extraction by dialect', () => { + expect(jsonArrayExpand('books.tags', sqlite)).to.equal('json_each(books.tags)') + expect(jsonArrayExpand('books.tags', postgres)).to.equal("jsonb_array_elements_text(COALESCE(books.tags::jsonb, '[]'::jsonb)) AS json_each(value)") + + expect(jsonPathText('payload', ['metadata', 'filename'], sqlite)).to.equal("json_extract(payload, '$.metadata.filename')") + expect(jsonPathText('payload', ['metadata', 'filename'], postgres)).to.equal("payload::jsonb #>> '{metadata,filename}'") + + expect(jsonPathNumber('payload', ['duration'], sqlite)).to.equal("json_extract(payload, '$.duration')") + expect(jsonPathNumber('payload', ['duration'], postgres)).to.equal("NULLIF(payload::jsonb #>> '{duration}', '')::double precision") + }) +}) From 5f7d94c74dfda3369b3256bfc7855469ecbbffda Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 16:58:07 -0500 Subject: [PATCH 02/38] fix migration metadata table detection on postgres --- server/managers/MigrationManager.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index 588771318..df0778849 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -215,7 +215,7 @@ class MigrationManager { async checkOrCreateMigrationsMetaTable() { const queryInterface = this.sequelize.getQueryInterface() - let migrationsMetaTableExists = await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE) + let migrationsMetaTableExists = await this.tableExists(MigrationManager.MIGRATIONS_META_TABLE) // If the table exists, check that the `version` and `maxVersion` rows exist if (migrationsMetaTableExists) { @@ -255,6 +255,20 @@ class MigrationManager { } } + async tableExists(tableName) { + const queryInterface = this.sequelize.getQueryInterface() + if (typeof queryInterface.tableExists === 'function') { + return queryInterface.tableExists(tableName) + } + + const tables = await queryInterface.showAllTables() + return tables.some((table) => { + if (typeof table === 'string') return table === tableName + if (table?.tableName) return table.tableName === tableName + return false + }) + } + extractVersionFromTag(tag) { if (!tag) return null const versionMatch = tag.match(/^v?(\d+\.\d+\.\d+)/) From 29646998fb224c67f1ee037da3006dc5d72a34e1 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:03:03 -0500 Subject: [PATCH 03/38] handle postgres alias casing in migration metadata --- server/managers/MigrationManager.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index df0778849..51ded7c89 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -202,15 +202,15 @@ class MigrationManager { const migrationsMetaTable = `"${MigrationManager.MIGRATIONS_META_TABLE}"` await this.checkOrCreateMigrationsMetaTable() - const [{ version }] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, { + const [versionRow] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, { type: Sequelize.QueryTypes.SELECT }) - this.databaseVersion = version + this.databaseVersion = versionRow?.version - const [{ maxVersion }] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, { + const [maxVersionRow] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, { type: Sequelize.QueryTypes.SELECT }) - this.maxVersion = maxVersion + this.maxVersion = maxVersionRow?.maxVersion || maxVersionRow?.maxversion } async checkOrCreateMigrationsMetaTable() { From ca2dac988d095614a8fb525176864a1bcc1bf444 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:07:10 -0500 Subject: [PATCH 04/38] disable identifier quoting for postgres compatibility --- server/Database.js | 3 ++- server/managers/MigrationManager.js | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/server/Database.js b/server/Database.js index 68f6e2729..3d48980c1 100644 --- a/server/Database.js +++ b/server/Database.js @@ -296,7 +296,8 @@ class Database { this.sequelize = new Sequelize(process.env.DATABASE_URL, { dialect: 'postgres', logging: logging, - benchmark: benchmark + benchmark: benchmark, + quoteIdentifiers: false }) } diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index 51ded7c89..99c89e155 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -199,7 +199,7 @@ class MigrationManager { } async fetchVersionsFromDatabase() { - const migrationsMetaTable = `"${MigrationManager.MIGRATIONS_META_TABLE}"` + const migrationsMetaTable = MigrationManager.MIGRATIONS_META_TABLE await this.checkOrCreateMigrationsMetaTable() const [versionRow] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, { @@ -219,7 +219,7 @@ class MigrationManager { // If the table exists, check that the `version` and `maxVersion` rows exist if (migrationsMetaTableExists) { - const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM "${MigrationManager.MIGRATIONS_META_TABLE}" WHERE key IN ('version', 'maxVersion')`, { + const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM ${MigrationManager.MIGRATIONS_META_TABLE} WHERE key IN ('version', 'maxVersion')`, { type: Sequelize.QueryTypes.SELECT }) if (count < 2) { @@ -247,7 +247,7 @@ class MigrationManager { allowNull: false } }) - await this.sequelize.query(`INSERT INTO "${MigrationManager.MIGRATIONS_META_TABLE}" (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, { + await this.sequelize.query(`INSERT INTO ${MigrationManager.MIGRATIONS_META_TABLE} (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, { replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0' }, type: Sequelize.QueryTypes.INSERT }) @@ -319,7 +319,7 @@ class MigrationManager { async updateMaxVersion() { try { - await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :maxVersion WHERE key = 'maxVersion'`, { + await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :maxVersion WHERE key = 'maxVersion'`, { replacements: { maxVersion: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) @@ -331,7 +331,7 @@ class MigrationManager { async updateDatabaseVersion() { try { - await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :version WHERE key = 'version'`, { + await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :version WHERE key = 'version'`, { replacements: { version: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) From a9f509d34232460c6af4b43948edf2474dcf656d Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:12:12 -0500 Subject: [PATCH 05/38] make sqlite to postgres migrator case-insensitive --- server/scripts/migrateSqliteToPostgres.js | 70 +++++++++++++++-------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js index 60acc11e2..f5c49a8ef 100644 --- a/server/scripts/migrateSqliteToPostgres.js +++ b/server/scripts/migrateSqliteToPostgres.js @@ -132,13 +132,24 @@ async function main() { `SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`, [PG_SCHEMA] ) - const pgTables = new Set(pgTablesRows.rows.map((row) => row.table_name)) - const tablesToMigrate = sqliteTables.filter((table) => pgTables.has(table)) + const pgTablesByLowerName = new Map(pgTablesRows.rows.map((row) => [row.table_name.toLowerCase(), row.table_name])) + + const tablesToMigrate = sqliteTables + .map((sqliteTable) => { + const postgresTable = pgTablesByLowerName.get(sqliteTable.toLowerCase()) + if (!postgresTable) return null + return { + sqliteTable, + postgresTable + } + }) + .filter(Boolean) + tablesToMigrate.sort((a, b) => { - const ai = preferredOrder.indexOf(a) - const bi = preferredOrder.indexOf(b) - if (ai === -1 && bi === -1) return a.localeCompare(b) + const ai = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === a.sqliteTable.toLowerCase()) + const bi = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === b.sqliteTable.toLowerCase()) + if (ai === -1 && bi === -1) return a.sqliteTable.localeCompare(b.sqliteTable) if (ai === -1) return 1 if (bi === -1) return -1 return ai - bi @@ -148,34 +159,46 @@ async function main() { throw new Error('No overlapping tables found between SQLite and PostgreSQL') } - console.log(`[migrate] tables to migrate: ${tablesToMigrate.join(', ')}`) + console.log(`[migrate] tables to migrate: ${tablesToMigrate.map((table) => table.sqliteTable).join(', ')}`) const pgColumnsByTable = new Map() - for (const table of tablesToMigrate) { + for (const { postgresTable } of tablesToMigrate) { const columnsResult = await pg.query( `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, - [PG_SCHEMA, table] + [PG_SCHEMA, postgresTable] ) - const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name, column])) - pgColumnsByTable.set(table, columnMap) + const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column])) + pgColumnsByTable.set(postgresTable, columnMap) } if (!DRY_RUN) { await pg.query('BEGIN') await pg.query('SET session_replication_role = replica') - const truncateList = tablesToMigrate.map((table) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`).join(', ') + const truncateList = tablesToMigrate.map(({ postgresTable }) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`).join(', ') await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`) console.log('[migrate] truncated target tables') } - for (const table of tablesToMigrate) { - const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(table)}`) - const pgColumns = pgColumnsByTable.get(table) - const insertColumns = rows.length ? Object.keys(rows[0]).filter((column) => pgColumns.has(column)) : [] + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(sqliteTable)}`) + const pgColumns = pgColumnsByTable.get(postgresTable) + const insertColumns = rows.length + ? Object.keys(rows[0]) + .map((sqliteColumn) => { + const pgColumn = pgColumns.get(sqliteColumn.toLowerCase()) + if (!pgColumn) return null + return { + sqliteColumn, + postgresColumn: pgColumn.column_name, + metadata: pgColumn + } + }) + .filter(Boolean) + : [] if (!rows.length || !insertColumns.length) { - console.log(`[migrate] ${table}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`) + console.log(`[migrate] ${sqliteTable}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`) continue } @@ -189,19 +212,18 @@ async function main() { for (const row of batchRows) { const placeholders = [] for (const column of insertColumns) { - const pgColumn = pgColumns.get(column) - params.push(convertValue(row[column], pgColumn)) + params.push(convertValue(row[column.sqliteColumn], column.metadata)) placeholders.push(`$${paramIndex++}`) } valuesSql.push(`(${placeholders.join(', ')})`) } - const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)} (${insertColumns.map(quoteIdent).join(', ')}) VALUES ${valuesSql.join(', ')}` + const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}` await pg.query(insertSql, params) } } - console.log(`[migrate] ${table}: ${rows.length} rows`) + console.log(`[migrate] ${sqliteTable}: ${rows.length} rows`) } if (!DRY_RUN) { @@ -211,11 +233,11 @@ async function main() { } const parity = [] - for (const table of tablesToMigrate) { - const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(table)}`) - const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`) + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(sqliteTable)}`) + const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`) parity.push({ - table, + table: sqliteTable, sqliteCount: Number(sqliteCountRow.count || 0), postgresCount: Number(pgCountResult.rows[0].count || 0) }) From 49fb17db07e8ae26faaa44b366c0faf9154d2a6a Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:15:45 -0500 Subject: [PATCH 06/38] harden json normalization in postgres migration script --- server/scripts/migrateSqliteToPostgres.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js index f5c49a8ef..e8b72881c 100644 --- a/server/scripts/migrateSqliteToPostgres.js +++ b/server/scripts/migrateSqliteToPostgres.js @@ -89,10 +89,22 @@ function normalizeBoolean(value) { function normalizeJson(value) { if (value === null || value === undefined || value === '') return null if (typeof value === 'object') return value + + const textValue = String(value) + try { - return JSON.parse(value) + const parsed = JSON.parse(textValue) + if (typeof parsed === 'string') { + try { + return JSON.parse(parsed) + } catch (error) { + return parsed + } + } + return parsed } catch (error) { - return value + // Keep non-JSON payloads as JSON string values so inserts remain valid. + return JSON.stringify(textValue) } } From 063984f76ee5b639ee920b9ccd93f79e7dff98f2 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:45:32 -0500 Subject: [PATCH 07/38] prevent lossy postgres migration edge cases --- server/Database.js | 5 + server/models/Book.js | 2 +- server/models/PlaybackSession.js | 2 +- server/scripts/migrateSqliteToPostgres.js | 255 ++++++++++++++++-- .../scripts/migrateSqliteToPostgres.test.js | 131 +++++++++ 5 files changed, 372 insertions(+), 23 deletions(-) create mode 100644 test/server/scripts/migrateSqliteToPostgres.test.js diff --git a/server/Database.js b/server/Database.js index 3d48980c1..3cbf32819 100644 --- a/server/Database.js +++ b/server/Database.js @@ -412,6 +412,11 @@ class Database { require('./models/CustomMetadataProvider').init(this.sequelize) require('./models/MediaItemShare').init(this.sequelize) + if (this.isPostgresDialect() && !force && !this.isNew) { + Logger.info('[Database] Skipping sequelize.sync for existing postgres schema') + return Promise.resolve() + } + return this.sequelize.sync({ force, alter: false }) } diff --git a/server/models/Book.js b/server/models/Book.js index d8ce6ff29..558950072 100644 --- a/server/models/Book.js +++ b/server/models/Book.js @@ -149,7 +149,7 @@ class Book extends Model { }, title: DataTypes.STRING, titleIgnorePrefix: DataTypes.STRING, - subtitle: DataTypes.STRING, + subtitle: DataTypes.TEXT, publishedYear: DataTypes.STRING, publishedDate: DataTypes.STRING, publisher: DataTypes.STRING, diff --git a/server/models/PlaybackSession.js b/server/models/PlaybackSession.js index 196fbda6c..47526d5f6 100644 --- a/server/models/PlaybackSession.js +++ b/server/models/PlaybackSession.js @@ -190,7 +190,7 @@ class PlaybackSession extends Model { currentTime: DataTypes.FLOAT, serverVersion: DataTypes.STRING, coverPath: DataTypes.STRING, - timeListening: DataTypes.INTEGER, + timeListening: DataTypes.FLOAT, mediaMetadata: DataTypes.JSON, date: DataTypes.STRING, dayOfWeek: DataTypes.STRING, diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js index e8b72881c..43dc439da 100644 --- a/server/scripts/migrateSqliteToPostgres.js +++ b/server/scripts/migrateSqliteToPostgres.js @@ -9,11 +9,6 @@ const PG_SCHEMA = process.env.PG_SCHEMA || 'public' const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500) const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true' -if (!DATABASE_URL) { - console.error('[migrate] DATABASE_URL is required') - process.exit(1) -} - const preferredOrder = [ 'migrationsMeta', 'SequelizeMeta', @@ -75,6 +70,120 @@ function sqliteGet(db, sql, params = []) { }) } +async function findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) { + const issues = [] + + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const pgColumns = pgColumnsByTable.get(postgresTable) + if (!pgColumns) continue + + for (const column of pgColumns.values()) { + if (column.data_type !== 'character varying' || !column.character_maximum_length) continue + + const sqliteColumn = column.column_name + const maxLength = Number(column.character_maximum_length) + const quotedTable = quoteIdent(sqliteTable) + const quotedColumn = quoteIdent(sqliteColumn) + + try { + const maxLengthRow = await sqliteGet( + sqliteDb, + `SELECT MAX(LENGTH(${quotedColumn})) AS maxLength FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL` + ) + const actualMaxLength = Number(maxLengthRow?.maxLength || 0) + if (actualMaxLength <= maxLength) continue + + const overCountRow = await sqliteGet( + sqliteDb, + `SELECT COUNT(*) AS count FROM ${quotedTable} WHERE LENGTH(${quotedColumn}) > ?`, + [maxLength] + ) + + issues.push({ + sqliteTable, + sqliteColumn, + postgresTable, + postgresColumn: column.column_name, + maxLength, + actualMaxLength, + overCount: Number(overCountRow?.count || 0) + }) + } catch (error) { + // Ignore columns missing in sqlite source table + } + } + } + + return issues +} + +function isIntegerCompatible(value) { + if (value === null || value === undefined) return true + + if (typeof value === 'number') { + return Number.isFinite(value) && Number.isInteger(value) + } + + if (typeof value === 'string') { + const trimmed = value.trim() + if (!trimmed) return false + if (!/^-?\d+$/.test(trimmed)) return false + const parsed = Number(trimmed) + return Number.isFinite(parsed) + } + + return false +} + +async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) { + const issues = [] + + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const pgColumns = pgColumnsByTable.get(postgresTable) + if (!pgColumns) continue + + for (const column of pgColumns.values()) { + const dataType = column.data_type + if (dataType !== 'smallint' && dataType !== 'integer' && dataType !== 'bigint') continue + + const sqliteColumn = column.column_name + const quotedTable = quoteIdent(sqliteTable) + const quotedColumn = quoteIdent(sqliteColumn) + + let rows + try { + rows = await sqliteAll(sqliteDb, `SELECT ${quotedColumn} AS value FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`) + } catch (error) { + // Ignore columns missing in sqlite source table + continue + } + + let badCount = 0 + let sampleValue = null + for (const row of rows) { + if (!isIntegerCompatible(row.value)) { + badCount += 1 + if (sampleValue === null) sampleValue = row.value + } + } + + if (badCount > 0) { + issues.push({ + sqliteTable, + sqliteColumn, + postgresTable, + postgresColumn: column.column_name, + postgresType: dataType, + badCount, + sampleValue + }) + } + } + } + + return issues +} + function normalizeBoolean(value) { if (value === null || value === undefined) return null if (typeof value === 'boolean') return value @@ -88,7 +197,7 @@ function normalizeBoolean(value) { function normalizeJson(value) { if (value === null || value === undefined || value === '') return null - if (typeof value === 'object') return value + if (typeof value === 'object') return JSON.stringify(value) const textValue = String(value) @@ -96,14 +205,14 @@ function normalizeJson(value) { const parsed = JSON.parse(textValue) if (typeof parsed === 'string') { try { - return JSON.parse(parsed) + return JSON.stringify(JSON.parse(parsed)) } catch (error) { - return parsed + return JSON.stringify(parsed) } } - return parsed + return JSON.stringify(parsed) } catch (error) { - // Keep non-JSON payloads as JSON string values so inserts remain valid. + // Keep non-JSON payloads as JSON string values so inserts remain valid JSON. return JSON.stringify(textValue) } } @@ -121,10 +230,45 @@ function convertValue(value, pgColumn) { return normalizeJson(value) } + if ((dataType === 'smallint' || dataType === 'integer' || dataType === 'bigint') && value !== null && value !== undefined) { + if (typeof value === 'number') return value + if (typeof value === 'string' && /^-?\d+$/.test(value.trim())) { + return Number(value) + } + } + return value } +function getOverlongColumns(row, insertColumns) { + const overlong = [] + + for (const column of insertColumns) { + const maxLength = column.metadata.character_maximum_length + if (!maxLength) continue + + const value = row[column.sqliteColumn] + if (value === null || value === undefined) continue + + const length = String(value).length + if (length > maxLength) { + overlong.push({ + sqliteColumn: column.sqliteColumn, + postgresColumn: column.postgresColumn, + maxLength, + actualLength: length + }) + } + } + + return overlong +} + async function main() { + if (!DATABASE_URL) { + throw new Error('DATABASE_URL is required') + } + console.log(`[migrate] sqlite source: ${SQLITE_PATH}`) console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`) console.log(`[migrate] dry run: ${DRY_RUN}`) @@ -176,13 +320,38 @@ async function main() { const pgColumnsByTable = new Map() for (const { postgresTable } of tablesToMigrate) { const columnsResult = await pg.query( - `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, + `SELECT column_name, data_type, udt_name, character_maximum_length FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, [PG_SCHEMA, postgresTable] ) const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column])) pgColumnsByTable.set(postgresTable, columnMap) } + const overlongVarcharIssues = await findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) + const integerTypeIssues = await findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) + + if (overlongVarcharIssues.length) { + console.error('[migrate] overlong source values detected for varchar columns:') + overlongVarcharIssues.forEach((issue) => { + console.error( + `[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` + + `(max=${issue.maxLength}, actualMax=${issue.actualMaxLength}, overRows=${issue.overCount})` + ) + }) + throw new Error('Migration aborted to prevent truncation/data loss. Widen target column types first.') + } + + if (integerTypeIssues.length) { + console.error('[migrate] non-integer source values detected for integer columns:') + integerTypeIssues.forEach((issue) => { + console.error( + `[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` + + `(type=${issue.postgresType}, badRows=${issue.badCount}, sample=${JSON.stringify(issue.sampleValue)})` + ) + }) + throw new Error('Migration aborted to prevent numeric precision loss. Widen target numeric column types first.') + } + if (!DRY_RUN) { await pg.query('BEGIN') await pg.query('SET session_replication_role = replica') @@ -222,16 +391,49 @@ async function main() { let paramIndex = 1 for (const row of batchRows) { - const placeholders = [] - for (const column of insertColumns) { - params.push(convertValue(row[column.sqliteColumn], column.metadata)) - placeholders.push(`$${paramIndex++}`) - } + const placeholders = [] + for (const column of insertColumns) { + params.push(convertValue(row[column.sqliteColumn], column.metadata)) + placeholders.push(`$${paramIndex++}`) + } valuesSql.push(`(${placeholders.join(', ')})`) } const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}` - await pg.query(insertSql, params) + + try { + await pg.query('SAVEPOINT migrate_batch') + await pg.query(insertSql, params) + await pg.query('RELEASE SAVEPOINT migrate_batch') + } catch (error) { + await pg.query('ROLLBACK TO SAVEPOINT migrate_batch') + console.error(`[migrate] batch insert failed for ${sqliteTable} (offset=${offset}, size=${batchRows.length}): ${error.message}`) + + for (let rowIndex = 0; rowIndex < batchRows.length; rowIndex++) { + const row = batchRows[rowIndex] + const singleRowParams = insertColumns.map((column) => convertValue(row[column.sqliteColumn], column.metadata)) + const singleRowInsertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES (${singleRowParams.map((_, index) => `$${index + 1}`).join(', ')})` + + try { + await pg.query('SAVEPOINT migrate_row') + await pg.query(singleRowInsertSql, singleRowParams) + await pg.query('RELEASE SAVEPOINT migrate_row') + } catch (rowError) { + await pg.query('ROLLBACK TO SAVEPOINT migrate_row') + const overlongColumns = getOverlongColumns(row, insertColumns) + if (overlongColumns.length) { + overlongColumns.forEach((column) => { + console.error( + `[migrate] overlong value in ${sqliteTable}.${column.sqliteColumn} -> ${postgresTable}.${column.postgresColumn} ` + + `(length=${column.actualLength}, max=${column.maxLength})` + ) + }) + } + + throw rowError + } + } + } } } @@ -271,7 +473,18 @@ async function main() { } } -main().catch((error) => { - console.error('[migrate] failed:', error) - process.exit(1) -}) +if (require.main === module) { + main().catch((error) => { + console.error('[migrate] failed:', error) + process.exit(1) + }) +} + +module.exports = { + normalizeJson, + isIntegerCompatible, + convertValue, + findOverlongVarcharValues, + findIntegerTypeIssues, + quoteIdent +} diff --git a/test/server/scripts/migrateSqliteToPostgres.test.js b/test/server/scripts/migrateSqliteToPostgres.test.js new file mode 100644 index 000000000..9b218079e --- /dev/null +++ b/test/server/scripts/migrateSqliteToPostgres.test.js @@ -0,0 +1,131 @@ +const { expect } = require('chai') +const sqlite3 = require('sqlite3') + +const { + normalizeJson, + isIntegerCompatible, + findOverlongVarcharValues, + findIntegerTypeIssues +} = require('../../../server/scripts/migrateSqliteToPostgres') + +function openMemoryDb() { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(':memory:', (error) => { + if (error) return reject(error) + resolve(db) + }) + }) +} + +function run(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.run(sql, params, (error) => { + if (error) return reject(error) + resolve() + }) + }) +} + +function close(db) { + return new Promise((resolve, reject) => { + db.close((error) => { + if (error) return reject(error) + resolve() + }) + }) +} + +describe('migrateSqliteToPostgres script helpers', () => { + it('should keep malformed JSON payloads insertable by returning valid JSON text', () => { + const malformed = '"{"x",1}"' + const normalized = normalizeJson(malformed) + + expect(() => JSON.parse(normalized)).to.not.throw() + expect(JSON.parse(normalized)).to.equal('"{"x",1}"') + }) + + it('should detect overlong varchar values before migration', async () => { + const db = await openMemoryDb() + + try { + await run(db, 'CREATE TABLE books (subtitle TEXT)') + await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['x'.repeat(300)]) + await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['ok']) + + const tablesToMigrate = [{ sqliteTable: 'books', postgresTable: 'books' }] + const pgColumnsByTable = new Map([ + [ + 'books', + new Map([ + [ + 'subtitle', + { + column_name: 'subtitle', + data_type: 'character varying', + character_maximum_length: 255 + } + ] + ]) + ] + ]) + + const issues = await findOverlongVarcharValues(db, tablesToMigrate, pgColumnsByTable) + + expect(issues).to.have.length(1) + expect(issues[0]).to.include({ + sqliteTable: 'books', + sqliteColumn: 'subtitle', + postgresTable: 'books', + postgresColumn: 'subtitle', + maxLength: 255, + actualMaxLength: 300, + overCount: 1 + }) + } finally { + await close(db) + } + }) + + it('should detect non-integer values for integer target columns', async () => { + const db = await openMemoryDb() + + try { + await run(db, 'CREATE TABLE playbackSessions (timeListening REAL)') + await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [25.802536999999997]) + await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [42]) + + const tablesToMigrate = [{ sqliteTable: 'playbackSessions', postgresTable: 'playbackSessions' }] + const pgColumnsByTable = new Map([ + [ + 'playbackSessions', + new Map([ + [ + 'timelistening', + { + column_name: 'timeListening', + data_type: 'integer' + } + ] + ]) + ] + ]) + + const issues = await findIntegerTypeIssues(db, tablesToMigrate, pgColumnsByTable) + + expect(issues).to.have.length(1) + expect(issues[0].sqliteTable).to.equal('playbackSessions') + expect(issues[0].sqliteColumn).to.equal('timeListening') + expect(issues[0].postgresType).to.equal('integer') + expect(issues[0].badCount).to.equal(1) + } finally { + await close(db) + } + }) + + it('should only accept integer-compatible values for integer columns', () => { + expect(isIntegerCompatible(10)).to.equal(true) + expect(isIntegerCompatible('10')).to.equal(true) + expect(isIntegerCompatible(10.5)).to.equal(false) + expect(isIntegerCompatible('10.5')).to.equal(false) + }) +}) From 93bf6d28a081c4892995426dfcf8dd1eb56ff134 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:50:08 -0500 Subject: [PATCH 08/38] fix postgres legacy user id lookup --- server/models/User.js | 4 +++- test/server/models/User.test.js | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 test/server/models/User.test.js diff --git a/server/models/User.js b/server/models/User.js index 0b9d49438..fa489fe14 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -414,9 +414,11 @@ class User extends Model { const cachedUser = userCache.getById(userId) || userCache.getByOldId(userId) if (cachedUser) return cachedUser + const idMatcher = this.sequelize.getDialect() === 'postgres' ? sequelize.where(sequelize.cast(sequelize.col('user.id'), 'text'), userId) : { id: userId } + const user = await this.findOne({ where: { - [sequelize.Op.or]: [{ id: userId }, { 'extraData.oldUserId': userId }] + [sequelize.Op.or]: [idMatcher, { 'extraData.oldUserId': userId }] }, include: this.sequelize.models.mediaProgress }) diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js new file mode 100644 index 000000000..9e2bf5888 --- /dev/null +++ b/test/server/models/User.test.js @@ -0,0 +1,42 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const sequelize = require('sequelize') + +const User = require('../../../server/models/User') + +describe('User model', () => { + describe('getUserByIdOrOldId', () => { + let originalSequelize + + beforeEach(() => { + originalSequelize = User.sequelize + }) + + afterEach(() => { + User.sequelize = originalSequelize + sinon.restore() + }) + + it('should cast id to text on postgres when checking legacy token ids', async () => { + User.sequelize = { + getDialect: () => 'postgres', + models: { + mediaProgress: {} + } + } + + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + await User.getUserByIdOrOldId('root') + + expect(findOneStub.calledOnce).to.equal(true) + + const options = findOneStub.firstCall.args[0] + const whereOr = options.where[sequelize.Op.or] + + expect(whereOr).to.have.length(2) + expect(whereOr[0].attribute.type).to.equal('text') + expect(whereOr[1]).to.deep.equal({ 'extraData.oldUserId': 'root' }) + }) + }) +}) From 1f4355b9d60c59100cd1c0c38e004d0f64b8e866 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 18:02:36 -0500 Subject: [PATCH 09/38] fix postgres old user id json lookup --- server/models/User.js | 20 ++++++++++--- test/server/models/User.test.js | 50 +++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/server/models/User.js b/server/models/User.js index fa489fe14..3442e6f1d 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -414,12 +414,24 @@ class User extends Model { const cachedUser = userCache.getById(userId) || userCache.getByOldId(userId) if (cachedUser) return cachedUser - const idMatcher = this.sequelize.getDialect() === 'postgres' ? sequelize.where(sequelize.cast(sequelize.col('user.id'), 'text'), userId) : { id: userId } + const isUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userId) + if (isUuid) { + const byId = await this.findByPk(userId, { + include: this.sequelize.models.mediaProgress + }) + if (byId) { + userCache.set(byId) + return byId + } + } + + const oldIdMatcher = + this.sequelize.getDialect() === 'postgres' + ? sequelize.where(sequelize.literal(`"extraData"#>>'{oldUserId}'`), userId) + : { 'extraData.oldUserId': userId } const user = await this.findOne({ - where: { - [sequelize.Op.or]: [idMatcher, { 'extraData.oldUserId': userId }] - }, + where: oldIdMatcher, include: this.sequelize.models.mediaProgress }) diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js index 9e2bf5888..ce187f93e 100644 --- a/test/server/models/User.test.js +++ b/test/server/models/User.test.js @@ -1,7 +1,5 @@ const { expect } = require('chai') const sinon = require('sinon') -const sequelize = require('sequelize') - const User = require('../../../server/models/User') describe('User model', () => { @@ -17,7 +15,7 @@ describe('User model', () => { sinon.restore() }) - it('should cast id to text on postgres when checking legacy token ids', async () => { + it('should resolve UUID ids via primary-key lookup on postgres', async () => { User.sequelize = { getDialect: () => 'postgres', models: { @@ -25,6 +23,46 @@ describe('User model', () => { } } + const user = { id: 'e8e677b2-da16-4220-ab67-443b7714caf9' } + const findByPkStub = sinon.stub(User, 'findByPk').resolves(user) + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + const result = await User.getUserByIdOrOldId('e8e677b2-da16-4220-ab67-443b7714caf9') + + expect(result).to.equal(user) + expect(findByPkStub.calledOnce).to.equal(true) + expect(findOneStub.called).to.equal(false) + }) + + it('should query legacy oldUserId with postgres-safe JSON matcher', async () => { + User.sequelize = { + getDialect: () => 'postgres', + models: { + mediaProgress: {} + } + } + + const findByPkStub = sinon.stub(User, 'findByPk').resolves(null) + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + await User.getUserByIdOrOldId('root') + + expect(findByPkStub.called).to.equal(false) + expect(findOneStub.calledOnce).to.equal(true) + + const options = findOneStub.firstCall.args[0] + expect(options.where.attribute.val).to.equal('"extraData"#>>\'{oldUserId}\'') + expect(options.where.logic).to.equal('root') + }) + + it('should keep sqlite oldUserId matcher unchanged', async () => { + User.sequelize = { + getDialect: () => 'sqlite', + models: { + mediaProgress: {} + } + } + const findOneStub = sinon.stub(User, 'findOne').resolves(null) await User.getUserByIdOrOldId('root') @@ -32,11 +70,7 @@ describe('User model', () => { expect(findOneStub.calledOnce).to.equal(true) const options = findOneStub.firstCall.args[0] - const whereOr = options.where[sequelize.Op.or] - - expect(whereOr).to.have.length(2) - expect(whereOr[0].attribute.type).to.equal('text') - expect(whereOr[1]).to.deep.equal({ 'extraData.oldUserId': 'root' }) + expect(options.where).to.deep.equal({ 'extraData.oldUserId': 'root' }) }) }) }) From 151bc1f74de91c696018d41bcf332f3fac08962f Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 18:20:58 -0500 Subject: [PATCH 10/38] guard numeric casts for postgres text fields --- server/models/Series.js | 3 +- .../utils/queries/libraryItemsBookFilters.js | 34 +++++++------------ server/utils/sqlDialectHelpers.js | 18 +++++++++- test/server/utils/sqlDialectHelpers.test.js | 12 ++++++- 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/server/models/Series.js b/server/models/Series.js index 28450a010..192378b7d 100644 --- a/server/models/Series.js +++ b/server/models/Series.js @@ -1,6 +1,7 @@ const { DataTypes, Model, where, fn, col, literal } = require('sequelize') const { getTitlePrefixAtEnd, getTitleIgnorePrefix } = require('../utils/index') +const { safeTextToDoubleExpression } = require('../utils/sqlDialectHelpers') class Series extends Model { constructor(values, options) { @@ -158,7 +159,7 @@ class Series extends Model { } } ], - order: [[literal('CAST(`bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]] + order: [[literal(`${safeTextToDoubleExpression(this.sequelize.getDialect() === 'postgres' ? '"bookSeries"."sequence"' : '`bookSeries.sequence`', this.sequelize)} ASC NULLS LAST`)]] }) } diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 31df0d735..de5f63d4c 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -6,7 +6,7 @@ const authorFilters = require('./authorFilters') const ShareManager = require('../../managers/ShareManager') const { profile } = require('../profiler') const stringifySequelizeQuery = require('../stringifySequelizeQuery') -const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand } = require('../sqlDialectHelpers') +const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand, safeTextToDoubleExpression, safeTextToIntegerExpression } = require('../sqlDialectHelpers') const countCache = new Map() module.exports = { @@ -237,7 +237,7 @@ module.exports = { } else if (group === 'publishedDecades') { const startYear = parseInt(value) const endYear = parseInt(value, 10) + 9 - mediaWhere = Sequelize.where(Sequelize.literal('CAST(publishedYear AS INTEGER)'), { + mediaWhere = Sequelize.where(Sequelize.literal(safeTextToIntegerExpression('publishedYear', Database.sequelize)), { [Sequelize.Op.between]: [startYear, endYear] }) } @@ -274,7 +274,7 @@ module.exports = { } else if (sortBy === 'media.duration') { return [['duration', dir]] } else if (sortBy === 'media.metadata.publishedYear') { - return [[Sequelize.literal('CAST(book.publishedYear AS INTEGER)'), dir]] + return [[Sequelize.literal(safeTextToIntegerExpression('book.publishedYear', Database.sequelize)), dir]] } else if (sortBy === 'media.metadata.authorNameLF') { // Sort by author name last first, secondary sort by title return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesLastFirst', Database.sequelize)), dir], getTitleOrder()] @@ -288,10 +288,8 @@ module.exports = { return [getTitleOrder()] } else if (sortBy === 'sequence') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' - if (Database.sequelize.getDialect() === 'postgres') { - return [[Sequelize.literal(`CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ${nullDir}`)]] - } - return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]] + const sequenceColumn = Database.sequelize.getDialect() === 'postgres' ? '"series.bookSeries"."sequence"' : '`series.bookSeries.sequence`' + return [[Sequelize.literal(`${safeTextToDoubleExpression(sequenceColumn, Database.sequelize)} ${nullDir}`)]] } else if (sortBy === 'progress') { return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]] } else if (sortBy === 'progress.createdAt') { @@ -331,11 +329,7 @@ module.exports = { } ], order: [ - Sequelize.literal( - Database.sequelize.getDialect() === 'postgres' - ? 'CAST("books.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST' - : 'CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST' - ) + Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"books.bookSeries"."sequence"' : '`books.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) ] }) const bookSeriesToInclude = [] @@ -529,11 +523,7 @@ module.exports = { if (sortBy !== 'sequence') { // Secondary sort by sequence sortOrder.push([ - Sequelize.literal( - Database.sequelize.getDialect() === 'postgres' - ? 'CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST' - : 'CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST' - ) + Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"series.bookSeries"."sequence"' : '`series.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) ]) } } else if (filterGroup === 'issues') { @@ -755,11 +745,11 @@ module.exports = { let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = ${booleanLiteral(false, Database.sequelize)} OR mp.isFinished IS NULL)` if (library.settings.onlyShowLaterBooksInContinueSeries) { - const castType = Database.sequelize.getDialect() === 'postgres' ? 'DOUBLE PRECISION' : 'FLOAT' - const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as ${castType}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)` + const safeSequenceExpr = safeTextToDoubleExpression('bs.sequence', Database.sequelize) + const maxSequenceQuery = `(SELECT max(${safeSequenceExpr}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)` includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence']) - booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as ${castType}) > ${maxSequenceQuery}` + booksNotFinishedQuery = booksNotFinishedQuery + ` AND ${safeSequenceExpr} > ${maxSequenceQuery}` } const { rows: series, count } = await Database.seriesModel.findAndCountAll({ @@ -794,7 +784,7 @@ module.exports = { attributes: ['bookId', 'sequence'], separate: true, subQuery: false, - order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], + order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]], where: { '$book.mediaProgresses.isFinished$': { [Sequelize.Op.or]: [null, false] @@ -907,7 +897,7 @@ module.exports = { model: Database.bookModel, where: userPermissionBookWhere.bookWhere }, - order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], + order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]], limit: 1 }, subQuery: false, diff --git a/server/utils/sqlDialectHelpers.js b/server/utils/sqlDialectHelpers.js index 36538bcb1..7f6f93118 100644 --- a/server/utils/sqlDialectHelpers.js +++ b/server/utils/sqlDialectHelpers.js @@ -61,6 +61,20 @@ function jsonPathNumber(columnExpression, pathSegments, sequelize) { return `json_extract(${columnExpression}, '$.${[].concat(pathSegments).join('.')}')` } +function safeTextToDoubleExpression(columnExpression, sequelize) { + if (isPostgres(sequelize)) { + return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(${columnExpression})::double precision ELSE NULL END` + } + return `CAST(${columnExpression} AS FLOAT)` +} + +function safeTextToIntegerExpression(columnExpression, sequelize) { + if (isPostgres(sequelize)) { + return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?\\d+$' THEN BTRIM(${columnExpression})::integer ELSE NULL END` + } + return `CAST(${columnExpression} AS INTEGER)` +} + module.exports = { getDialect, isPostgres, @@ -71,5 +85,7 @@ module.exports = { jsonArrayContainsValue, jsonArrayExpand, jsonPathText, - jsonPathNumber + jsonPathNumber, + safeTextToDoubleExpression, + safeTextToIntegerExpression } diff --git a/test/server/utils/sqlDialectHelpers.test.js b/test/server/utils/sqlDialectHelpers.test.js index 006b02364..82d9154d8 100644 --- a/test/server/utils/sqlDialectHelpers.test.js +++ b/test/server/utils/sqlDialectHelpers.test.js @@ -8,7 +8,9 @@ const { jsonArrayContainsValue, jsonArrayExpand, jsonPathText, - jsonPathNumber + jsonPathNumber, + safeTextToDoubleExpression, + safeTextToIntegerExpression } = require('../../../server/utils/sqlDialectHelpers') const sqlite = { @@ -55,4 +57,12 @@ describe('sqlDialectHelpers', () => { expect(jsonPathNumber('payload', ['duration'], sqlite)).to.equal("json_extract(payload, '$.duration')") expect(jsonPathNumber('payload', ['duration'], postgres)).to.equal("NULLIF(payload::jsonb #>> '{duration}', '')::double precision") }) + + it('should generate safe numeric cast expressions for sequence-like text', () => { + expect(safeTextToDoubleExpression('sequence', sqlite)).to.equal('CAST(sequence AS FLOAT)') + expect(safeTextToDoubleExpression('"bookSeries"."sequence"', postgres)).to.equal("CASE WHEN BTRIM(\"bookSeries\".\"sequence\") ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(\"bookSeries\".\"sequence\")::double precision ELSE NULL END") + + expect(safeTextToIntegerExpression('publishedYear', sqlite)).to.equal('CAST(publishedYear AS INTEGER)') + expect(safeTextToIntegerExpression('book.publishedYear', postgres)).to.equal("CASE WHEN BTRIM(book.publishedYear) ~ '^[+-]?\\d+$' THEN BTRIM(book.publishedYear)::integer ELSE NULL END") + }) }) From f23cd0ba59a9f070ee76f0b853952e3fa7c9bf44 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 19:33:47 -0500 Subject: [PATCH 11/38] fix postgres series join alias in sequence sort --- server/utils/queries/libraryItemsBookFilters.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index de5f63d4c..346d775a0 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -288,7 +288,7 @@ module.exports = { return [getTitleOrder()] } else if (sortBy === 'sequence') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' - const sequenceColumn = Database.sequelize.getDialect() === 'postgres' ? '"series.bookSeries"."sequence"' : '`series.bookSeries.sequence`' + const sequenceColumn = Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`' return [[Sequelize.literal(`${safeTextToDoubleExpression(sequenceColumn, Database.sequelize)} ${nullDir}`)]] } else if (sortBy === 'progress') { return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]] @@ -329,7 +329,7 @@ module.exports = { } ], order: [ - Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"books.bookSeries"."sequence"' : '`books.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) + Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"books->bookSeries"."sequence"' : '`books.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) ] }) const bookSeriesToInclude = [] @@ -523,7 +523,7 @@ module.exports = { if (sortBy !== 'sequence') { // Secondary sort by sequence sortOrder.push([ - Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"series.bookSeries"."sequence"' : '`series.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) + Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) ]) } } else if (filterGroup === 'issues') { From 78320963dee742a9449f84553422446e215f3244 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 19:46:21 -0500 Subject: [PATCH 12/38] fix postgres collapse-series id quoting --- server/utils/queries/libraryItemsBookFilters.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 346d775a0..02258e85b 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -619,11 +619,12 @@ module.exports = { const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."title"' : '`libraryItem`.`title`' const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."titleIgnorePrefix"' : '`libraryItem`.`titleIgnorePrefix`' const fallbackFn = coalesceFunctionName(Database.sequelize) + const includedBookSeriesIds = bookSeriesToInclude.map((v) => Database.sequelize.escape(v.id)).join(', ') if (global.ServerSettings.sortingIgnorePrefix) { - bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds})), ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) } else { - bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitle})`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds})), ${fallbackLibraryItemTitle})`), 'display_title']) } } From 97e4adae8d966e711f425ceb84716026bf001167 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 19:51:07 -0500 Subject: [PATCH 13/38] fix postgres media progress timestamp update --- server/models/MediaProgress.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/models/MediaProgress.js b/server/models/MediaProgress.js index 9c0269a9e..e562b7109 100644 --- a/server/models/MediaProgress.js +++ b/server/models/MediaProgress.js @@ -257,7 +257,13 @@ class MediaProgress extends Model { const escapedDate = this.sequelize.escape(new Date(progressPayload.lastUpdate)) Logger.info(`[MediaProgress] Manually setting updatedAt to ${escapedDate} (media item ${this.mediaItemId})`) - await this.sequelize.query(`UPDATE "mediaProgresses" SET "updatedAt" = ${escapedDate} WHERE "id" = '${this.id}'`) + await this.constructor.update( + { updatedAt: new Date(progressPayload.lastUpdate) }, + { + where: { id: this.id }, + silent: true + } + ) await this.reload() } From c5d85614e08a21ee632efca9165d63b324fab734 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 19:55:23 -0500 Subject: [PATCH 14/38] fix postgres oldUserId column casing --- server/models/User.js | 2 +- test/server/models/User.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/models/User.js b/server/models/User.js index 3442e6f1d..633fc1cef 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -427,7 +427,7 @@ class User extends Model { const oldIdMatcher = this.sequelize.getDialect() === 'postgres' - ? sequelize.where(sequelize.literal(`"extraData"#>>'{oldUserId}'`), userId) + ? sequelize.where(sequelize.literal(`extradata#>>'{oldUserId}'`), userId) : { 'extraData.oldUserId': userId } const user = await this.findOne({ diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js index ce187f93e..82c289c45 100644 --- a/test/server/models/User.test.js +++ b/test/server/models/User.test.js @@ -51,7 +51,7 @@ describe('User model', () => { expect(findOneStub.calledOnce).to.equal(true) const options = findOneStub.firstCall.args[0] - expect(options.where.attribute.val).to.equal('"extraData"#>>\'{oldUserId}\'') + expect(options.where.attribute.val).to.equal("extradata#>>'{oldUserId}'") expect(options.where.logic).to.equal('root') }) From 6021a18aa048c052cee0cde6eb69b0447d8fcb4f Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 20:15:03 -0500 Subject: [PATCH 15/38] fix empty collapse-series list query and add diagnostics --- .../utils/queries/libraryItemsBookFilters.js | 61 +++++++++++++------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 02258e85b..a0721a920 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -362,26 +362,32 @@ module.exports = { async findAndCountAll(findOptions, limit, offset, useCountCache) { const model = Database.bookModel - if (useCountCache) { - const countCacheKey = stringifySequelizeQuery(findOptions) - Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`) - if (!countCache.has(countCacheKey)) { - const count = await model.count(findOptions) - countCache.set(countCacheKey, count) - } - - findOptions.limit = limit || null - findOptions.offset = offset - - const rows = await model.findAll(findOptions) - - return { rows, count: countCache.get(countCacheKey) } - } - findOptions.limit = limit || null findOptions.offset = offset - return await model.findAndCountAll(findOptions) + try { + if (useCountCache) { + const countCacheKey = stringifySequelizeQuery(findOptions) + Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`) + if (!countCache.has(countCacheKey)) { + const count = await model.count(findOptions) + countCache.set(countCacheKey, count) + } + + const rows = await model.findAll(findOptions) + return { rows, count: countCache.get(countCacheKey) } + } + + return await model.findAndCountAll(findOptions) + } catch (error) { + Logger.error(`[LibraryItemsBookFilters] findAndCountAll failed: ${error.message}`) + try { + Logger.error(`[LibraryItemsBookFilters] findAndCountAll query: ${stringifySequelizeQuery(findOptions)}`) + } catch (stringifyError) { + Logger.error(`[LibraryItemsBookFilters] failed to stringify query: ${stringifyError.message}`) + } + throw error + } }, /** @@ -620,11 +626,28 @@ module.exports = { const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."titleIgnorePrefix"' : '`libraryItem`.`titleIgnorePrefix`' const fallbackFn = coalesceFunctionName(Database.sequelize) const includedBookSeriesIds = bookSeriesToInclude.map((v) => Database.sequelize.escape(v.id)).join(', ') + const collapseSeriesSubqueryByName = includedBookSeriesIds + ? `(SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))` + : 'NULL' + const collapseSeriesSubqueryByNameIgnorePrefix = includedBookSeriesIds + ? `(SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))` + : 'NULL' + + Logger.debug( + `[LibraryItemsBookFilters] collapse-series computed includeIds=${bookSeriesToInclude.length} excludeBooks=${booksToExclude.length} ` + + `filterGroup=${filterGroup || 'none'} filterValue=${filterValue || 'none'} sortBy=${sortBy}` + ) + if (!includedBookSeriesIds) { + Logger.warn( + `[LibraryItemsBookFilters] collapse-series produced no include IDs; using library item title fallback ` + + `(libraryId=${libraryId}, filterGroup=${filterGroup || 'none'}, filterValue=${filterValue || 'none'}, sortBy=${sortBy})` + ) + } if (global.ServerSettings.sortingIgnorePrefix) { - bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds})), ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByNameIgnorePrefix}, ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) } else { - bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds})), ${fallbackLibraryItemTitle})`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByName}, ${fallbackLibraryItemTitle})`), 'display_title']) } } From 985e7ffa1aa1bda484793086781a496782554a3d Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 20:32:12 -0500 Subject: [PATCH 16/38] fix postgres collapsed-series display title alias --- server/utils/queries/libraryItemsBookFilters.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index a0721a920..e94e3d608 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -622,8 +622,8 @@ module.exports = { // When collapsing series and sorting by title then use the series name instead of the book title // for this set an attribute "display_title" to use in sorting - const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."title"' : '`libraryItem`.`title`' - const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."titleIgnorePrefix"' : '`libraryItem`.`titleIgnorePrefix`' + const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.title' : '`libraryItem`.`title`' + const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.titleIgnorePrefix' : '`libraryItem`.`titleIgnorePrefix`' const fallbackFn = coalesceFunctionName(Database.sequelize) const includedBookSeriesIds = bookSeriesToInclude.map((v) => Database.sequelize.escape(v.id)).join(', ') const collapseSeriesSubqueryByName = includedBookSeriesIds From a7554ac032ab2148de9d523c319b8c2d9176aafd Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 22:08:15 -0500 Subject: [PATCH 17/38] add regression tests for postgres query edge cases --- test/server/models/MediaProgress.test.js | 70 +++++++++ .../scripts/migrateSqliteToPostgres.test.js | 15 ++ .../queries/libraryItemsBookFilters.test.js | 133 ++++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 test/server/models/MediaProgress.test.js create mode 100644 test/server/utils/queries/libraryItemsBookFilters.test.js diff --git a/test/server/models/MediaProgress.test.js b/test/server/models/MediaProgress.test.js new file mode 100644 index 000000000..a7c490306 --- /dev/null +++ b/test/server/models/MediaProgress.test.js @@ -0,0 +1,70 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Logger = require('../../../server/Logger') +const MediaProgress = require('../../../server/models/MediaProgress') + +function createProgressInstance() { + const progress = Object.create(MediaProgress.prototype) + + Object.defineProperties(progress, { + id: { value: 'progress-1', writable: true, configurable: true }, + mediaItemId: { value: 'media-1', writable: true, configurable: true }, + duration: { value: 3600, writable: true, configurable: true }, + currentTime: { value: 120, writable: true, configurable: true }, + isFinished: { value: false, writable: true, configurable: true }, + hideFromContinueListening: { value: false, writable: true, configurable: true }, + extraData: { value: {}, writable: true, configurable: true } + }) + + progress.changed = sinon.stub().returns(false) + progress.set = sinon.stub().callsFake((payload) => Object.assign(progress, payload)) + progress.save = sinon.stub().resolves() + progress.reload = sinon.stub().resolves() + progress.constructor = { + update: sinon.stub().resolves(), + sequelize: { + escape: (value) => `'${value.toISOString ? value.toISOString() : value}'` + } + } + + return progress +} + +describe('MediaProgress', () => { + afterEach(() => { + sinon.restore() + }) + + describe('applyProgressUpdate', () => { + it('should update updatedAt via model update for valid lastUpdate', async () => { + const progress = createProgressInstance() + const infoStub = sinon.stub(Logger, 'info') + const lastUpdate = '2026-03-03T01:00:00.000Z' + + await progress.applyProgressUpdate({ currentTime: 130, lastUpdate }) + + expect(progress.save.calledOnce).to.equal(true) + expect(progress.constructor.update.calledOnce).to.equal(true) + expect(progress.constructor.update.firstCall.args[0].updatedAt.toISOString()).to.equal(new Date(lastUpdate).toISOString()) + expect(progress.constructor.update.firstCall.args[1]).to.deep.equal({ + where: { id: 'progress-1' }, + silent: true + }) + expect(progress.reload.calledOnce).to.equal(true) + expect(infoStub.calledWithMatch('[MediaProgress] Manually setting updatedAt')).to.equal(true) + }) + + it('should skip manual updatedAt update when lastUpdate is invalid', async () => { + const progress = createProgressInstance() + const warnStub = sinon.stub(Logger, 'warn') + + await progress.applyProgressUpdate({ currentTime: 130, lastUpdate: 'invalid-date' }) + + expect(progress.save.calledOnce).to.equal(true) + expect(progress.constructor.update.called).to.equal(false) + expect(progress.reload.called).to.equal(false) + expect(warnStub.calledWithMatch('[MediaProgress] Invalid date provided for lastUpdate')).to.equal(true) + }) + }) +}) diff --git a/test/server/scripts/migrateSqliteToPostgres.test.js b/test/server/scripts/migrateSqliteToPostgres.test.js index 9b218079e..d0bb3d592 100644 --- a/test/server/scripts/migrateSqliteToPostgres.test.js +++ b/test/server/scripts/migrateSqliteToPostgres.test.js @@ -4,6 +4,7 @@ const sqlite3 = require('sqlite3') const { normalizeJson, isIntegerCompatible, + convertValue, findOverlongVarcharValues, findIntegerTypeIssues } = require('../../../server/scripts/migrateSqliteToPostgres') @@ -128,4 +129,18 @@ describe('migrateSqliteToPostgres script helpers', () => { expect(isIntegerCompatible(10.5)).to.equal(false) expect(isIntegerCompatible('10.5')).to.equal(false) }) + + it('should coerce integer-like strings for postgres integer columns', () => { + expect(convertValue('42', { data_type: 'integer' })).to.equal(42) + expect(convertValue('-7', { data_type: 'bigint' })).to.equal(-7) + expect(convertValue('4.2', { data_type: 'integer' })).to.equal('4.2') + }) + + it('should always return valid json text for postgres json columns', () => { + const convertedObject = convertValue({ a: 1 }, { data_type: 'jsonb', udt_name: 'jsonb' }) + const convertedMalformed = convertValue('"{"x",1}"', { data_type: 'jsonb', udt_name: 'jsonb' }) + + expect(JSON.parse(convertedObject)).to.deep.equal({ a: 1 }) + expect(JSON.parse(convertedMalformed)).to.equal('"{"x",1}"') + }) }) diff --git a/test/server/utils/queries/libraryItemsBookFilters.test.js b/test/server/utils/queries/libraryItemsBookFilters.test.js new file mode 100644 index 000000000..b014c1399 --- /dev/null +++ b/test/server/utils/queries/libraryItemsBookFilters.test.js @@ -0,0 +1,133 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Database = require('../../../../server/Database') +const Logger = require('../../../../server/Logger') +const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters') + +function createSequelizeStub(dialect, models) { + return { + getDialect: () => dialect, + escape: (value) => `'${String(value).replace(/'/g, "''")}'`, + random: () => ({ fn: 'random' }), + models + } +} + +function createModelStubs() { + return { + book: { + findAndCountAll: sinon.stub(), + findAll: sinon.stub(), + count: sinon.stub() + }, + libraryItem: {}, + feed: {}, + bookSeries: {}, + series: { + findAll: sinon.stub() + }, + bookAuthor: {}, + author: {}, + mediaProgress: {}, + mediaItemShare: {} + } +} + +describe('libraryItemsBookFilters postgres query safety', () => { + let originalSequelize + let originalServerSettings + let modelStubs + + beforeEach(() => { + originalSequelize = Database.sequelize + originalServerSettings = global.ServerSettings + modelStubs = createModelStubs() + global.ServerSettings = { + sortingIgnorePrefix: false + } + }) + + afterEach(() => { + Database.sequelize = originalSequelize + global.ServerSettings = originalServerSettings + sinon.restore() + }) + + it('should avoid generating IN () when collapse-series has no include ids', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 }) + + sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({ + booksToExclude: [], + bookSeriesToInclude: [] + }) + const warnStub = sinon.stub(Logger, 'warn') + + await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, true, [], 20, 0) + + const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0] + const displayTitleExpression = findOptions.attributes.include[0][0].val + + expect(displayTitleExpression).to.include('COALESCE(NULL, libraryItem.title)') + expect(displayTitleExpression).to.not.include('IN ()') + expect(warnStub.calledOnce).to.equal(true) + }) + + it('should escape collapse-series ids safely for postgres subquery', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 }) + + sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({ + booksToExclude: [], + bookSeriesToInclude: [{ id: "series-id-'quoted'", numBooks: 2, libraryItemIds: [] }] + }) + global.ServerSettings.sortingIgnorePrefix = true + + await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.title', false, true, [], 20, 0) + + const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0] + const displayTitleExpression = findOptions.attributes.include[0][0].val + + expect(displayTitleExpression).to.include("bs.id IN ('series-id-''quoted''')") + expect(displayTitleExpression).to.include('libraryItem.titleIgnorePrefix') + }) + + it('should use postgres-safe join alias for sequence sorting', () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + + const order = libraryItemsBookFilters.getOrder('sequence', false, false) + const expression = order[0][0].val + + expect(expression).to.include('"series->bookSeries"."sequence"') + expect(expression).to.include('CASE WHEN BTRIM') + }) + + it('should log generated query when findAndCountAll fails', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.book.findAndCountAll.rejects(new Error('boom')) + const errorStub = sinon.stub(Logger, 'error') + + try { + await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, false, [], 20, 0) + expect.fail('Expected getFilteredLibraryItems to throw') + } catch (error) { + expect(error.message).to.equal('boom') + } + + expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll failed: boom')).to.equal(true) + expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll query:')).to.equal(true) + }) + + it('should use postgres-safe books->bookSeries alias in collapse-series selector query', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.series.findAll.resolves([]) + + await libraryItemsBookFilters.getCollapseSeriesBooksToExclude({ where: {}, include: [] }, null) + + const findAllOptions = modelStubs.series.findAll.firstCall.args[0] + const orderExpression = findAllOptions.order[0].val + + expect(orderExpression).to.include('"books->bookSeries"."sequence"') + }) +}) From c757f96a88035af2fd9f77ecc1986fb822563173 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Tue, 3 Mar 2026 15:12:38 -0500 Subject: [PATCH 18/38] tighten postgres compatibility tests and diagnostics --- .../utils/queries/libraryItemsBookFilters.js | 2 +- test/server/Database.test.js | 101 +++++++++++++++++ test/server/managers/MigrationManager.test.js | 60 ++++++++++ test/server/models/User.test.js | 20 ++++ .../queries/libraryItemsBookFilters.test.js | 4 +- .../libraryItemsPodcastFilters.test.js | 75 ++++++++++++ .../utils/queries/seriesFilters.test.js | 107 ++++++++++++++++++ 7 files changed, 366 insertions(+), 3 deletions(-) create mode 100644 test/server/Database.test.js create mode 100644 test/server/utils/queries/libraryItemsPodcastFilters.test.js create mode 100644 test/server/utils/queries/seriesFilters.test.js diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index e94e3d608..0a5a6a4ed 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -638,7 +638,7 @@ module.exports = { `filterGroup=${filterGroup || 'none'} filterValue=${filterValue || 'none'} sortBy=${sortBy}` ) if (!includedBookSeriesIds) { - Logger.warn( + Logger.debug( `[LibraryItemsBookFilters] collapse-series produced no include IDs; using library item title fallback ` + `(libraryId=${libraryId}, filterGroup=${filterGroup || 'none'}, filterValue=${filterValue || 'none'}, sortBy=${sortBy})` ) diff --git a/test/server/Database.test.js b/test/server/Database.test.js new file mode 100644 index 000000000..a70e1d204 --- /dev/null +++ b/test/server/Database.test.js @@ -0,0 +1,101 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const fs = require('../../server/libs/fsExtra') +const Database = require('../../server/Database') + +describe('Database', () => { + let originalDialect + let originalSequelize + let originalEnv + + beforeEach(() => { + originalDialect = Database.dialect + originalSequelize = Database.sequelize + originalEnv = { + DB_DIALECT: process.env.DB_DIALECT, + DATABASE_URL: process.env.DATABASE_URL + } + }) + + afterEach(() => { + Database.dialect = originalDialect + Database.sequelize = originalSequelize + + if (originalEnv.DB_DIALECT === undefined) delete process.env.DB_DIALECT + else process.env.DB_DIALECT = originalEnv.DB_DIALECT + + if (originalEnv.DATABASE_URL === undefined) delete process.env.DATABASE_URL + else process.env.DATABASE_URL = originalEnv.DATABASE_URL + + sinon.restore() + }) + + describe('getConfiguredDialect', () => { + it('should default to sqlite when no env variables are set', () => { + delete process.env.DB_DIALECT + delete process.env.DATABASE_URL + + expect(Database.getConfiguredDialect()).to.equal('sqlite') + }) + + it('should use explicit DB_DIALECT value when valid', () => { + process.env.DB_DIALECT = 'postgres' + process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite' + + expect(Database.getConfiguredDialect()).to.equal('postgres') + }) + + it('should infer postgres dialect from DATABASE_URL', () => { + delete process.env.DB_DIALECT + process.env.DATABASE_URL = 'postgresql://user:pass@localhost:5432/abs' + + expect(Database.getConfiguredDialect()).to.equal('postgres') + }) + + it('should fallback to sqlite for unsupported DB_DIALECT', () => { + process.env.DB_DIALECT = 'mysql' + process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite' + + expect(Database.getConfiguredDialect()).to.equal('sqlite') + }) + }) + + describe('checkHasDb', () => { + it('should not check sqlite file existence in postgres mode', async () => { + Database.dialect = 'postgres' + const pathExistsStub = sinon.stub(fs, 'pathExists') + + const hasDb = await Database.checkHasDb() + + expect(hasDb).to.equal(true) + expect(pathExistsStub.called).to.equal(false) + }) + }) + + describe('checkHasTables', () => { + it('should return true when at least one table exists', async () => { + Database.sequelize = { + getQueryInterface: () => ({ + showAllTables: sinon.stub().resolves(['users']) + }) + } + + const hasTables = await Database.checkHasTables() + + expect(hasTables).to.equal(true) + }) + + it('should return false when no tables exist', async () => { + Database.sequelize = { + getQueryInterface: () => ({ + showAllTables: sinon.stub().resolves([]) + }) + } + + const hasTables = await Database.checkHasTables() + + expect(hasTables).to.equal(false) + }) + }) +}) diff --git a/test/server/managers/MigrationManager.test.js b/test/server/managers/MigrationManager.test.js index af2e9da8f..a306eb5be 100644 --- a/test/server/managers/MigrationManager.test.js +++ b/test/server/managers/MigrationManager.test.js @@ -189,6 +189,52 @@ describe('MigrationManager', () => { expect(loggerInfoStub.calledWith(sinon.match('Restored the original database'))).to.be.true expect(processExitStub.calledOnce).to.be.true }) + + it('should skip sqlite backup workflow for postgres migrations', async () => { + // Arrange + migrationManager.serverVersion = '1.2.0' + migrationManager.databaseVersion = '1.1.0' + migrationManager.maxVersion = '1.1.0' + migrationManager.initialized = true + sequelizeStub.getDialect.returns('postgres') + + umzugStub.migrations.resolves([{ name: 'v1.2.0-migration.js' }]) + umzugStub.executed.resolves([{ name: 'v1.1.0-migration.js' }]) + + // Act + await migrationManager.runMigrations() + + // Assert + expect(umzugStub.up.calledOnce).to.be.true + expect(fsCopyStub.called).to.be.false + expect(fsRemoveStub.called).to.be.false + }) + }) + + describe('tableExists', () => { + it('should use queryInterface.tableExists when available', async () => { + const tableExistsStub = sinon.stub().resolves(true) + const sequelize = sinon.createStubInstance(Sequelize) + sequelize.getQueryInterface.returns({ tableExists: tableExistsStub }) + const manager = new MigrationManager(sequelize, false, configPath) + + const exists = await manager.tableExists('migrationsMeta') + + expect(exists).to.equal(true) + expect(tableExistsStub.calledOnceWithExactly('migrationsMeta')).to.equal(true) + }) + + it('should fallback to showAllTables and support object table names', async () => { + const sequelize = sinon.createStubInstance(Sequelize) + sequelize.getQueryInterface.returns({ + showAllTables: sinon.stub().resolves([{ tableName: 'migrationsMeta' }]) + }) + const manager = new MigrationManager(sequelize, false, configPath) + + const exists = await manager.tableExists('migrationsMeta') + + expect(exists).to.equal(true) + }) }) describe('fetchVersionsFromDatabase', () => { @@ -280,6 +326,20 @@ describe('MigrationManager', () => { expect(error.message).to.equal('Database query failed') } }) + + it('should support lowercase maxversion alias from postgres drivers', async () => { + const sequelize = sinon.createStubInstance(Sequelize) + sequelize.query.onFirstCall().resolves([{ version: '1.1.0' }]) + sequelize.query.onSecondCall().resolves([{ maxversion: '1.2.0' }]) + + const manager = new MigrationManager(sequelize, false, configPath) + manager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves() + + await manager.fetchVersionsFromDatabase() + + expect(manager.databaseVersion).to.equal('1.1.0') + expect(manager.maxVersion).to.equal('1.2.0') + }) }) describe('updateMaxVersion', () => { diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js index 82c289c45..5b935fee6 100644 --- a/test/server/models/User.test.js +++ b/test/server/models/User.test.js @@ -34,6 +34,26 @@ describe('User model', () => { expect(findOneStub.called).to.equal(false) }) + it('should resolve uppercase UUID ids via primary-key lookup on postgres', async () => { + User.sequelize = { + getDialect: () => 'postgres', + models: { + mediaProgress: {} + } + } + + const uppercaseUuid = 'E8E677B2-DA16-4220-AB67-443B7714CAF9' + const user = { id: uppercaseUuid } + const findByPkStub = sinon.stub(User, 'findByPk').resolves(user) + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + const result = await User.getUserByIdOrOldId(uppercaseUuid) + + expect(result).to.equal(user) + expect(findByPkStub.calledOnceWithExactly(uppercaseUuid, { include: User.sequelize.models.mediaProgress })).to.equal(true) + expect(findOneStub.called).to.equal(false) + }) + it('should query legacy oldUserId with postgres-safe JSON matcher', async () => { User.sequelize = { getDialect: () => 'postgres', diff --git a/test/server/utils/queries/libraryItemsBookFilters.test.js b/test/server/utils/queries/libraryItemsBookFilters.test.js index b014c1399..19b6a58bc 100644 --- a/test/server/utils/queries/libraryItemsBookFilters.test.js +++ b/test/server/utils/queries/libraryItemsBookFilters.test.js @@ -62,7 +62,7 @@ describe('libraryItemsBookFilters postgres query safety', () => { booksToExclude: [], bookSeriesToInclude: [] }) - const warnStub = sinon.stub(Logger, 'warn') + const debugStub = sinon.stub(Logger, 'debug') await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, true, [], 20, 0) @@ -71,7 +71,7 @@ describe('libraryItemsBookFilters postgres query safety', () => { expect(displayTitleExpression).to.include('COALESCE(NULL, libraryItem.title)') expect(displayTitleExpression).to.not.include('IN ()') - expect(warnStub.calledOnce).to.equal(true) + expect(debugStub.calledWithMatch('collapse-series produced no include IDs')).to.equal(true) }) it('should escape collapse-series ids safely for postgres subquery', async () => { diff --git a/test/server/utils/queries/libraryItemsPodcastFilters.test.js b/test/server/utils/queries/libraryItemsPodcastFilters.test.js new file mode 100644 index 000000000..79938abdc --- /dev/null +++ b/test/server/utils/queries/libraryItemsPodcastFilters.test.js @@ -0,0 +1,75 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Database = require('../../../../server/Database') +const libraryItemsPodcastFilters = require('../../../../server/utils/queries/libraryItemsPodcastFilters') + +describe('libraryItemsPodcastFilters dialect behavior', () => { + let originalSequelize + let originalServerSettings + + beforeEach(() => { + originalSequelize = Database.sequelize + originalServerSettings = global.ServerSettings + global.ServerSettings = { sortingIgnorePrefix: false } + }) + + afterEach(() => { + Database.sequelize = originalSequelize + global.ServerSettings = originalServerSettings + sinon.restore() + }) + + it('should build postgres json tag permission predicate', () => { + Database.sequelize = { getDialect: () => 'postgres' } + + const { podcastWhere, replacements } = libraryItemsPodcastFilters.getUserPermissionPodcastWhereQuery({ + canAccessExplicitContent: true, + permissions: { + accessAllTags: false, + itemTagsSelected: ['fiction'], + selectedTagsNotAccessible: false + } + }) + + expect(replacements.userTagsSelected).to.deep.equal(['fiction']) + expect(podcastWhere[0].attribute.val).to.include('jsonb_array_elements_text') + }) + + it('should build sqlite no-case author sort expression', () => { + Database.sequelize = { getDialect: () => 'sqlite' } + + const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false) + + expect(order[0][0].val).to.include('podcast.author COLLATE NOCASE') + }) + + it('should build postgres lower author sort expression', () => { + Database.sequelize = { getDialect: () => 'postgres' } + + const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false) + + expect(order[0][0].val).to.include('LOWER(podcast.author)') + }) + + it('should use postgres json duration extraction in podcast stats query', async () => { + const queryStub = sinon.stub() + queryStub.onFirstCall().resolves([[{ totalSize: 1000 }]]) + queryStub.onSecondCall().resolves([[{ totalDuration: '12.3', totalItems: '4', numAudioFiles: '9' }]]) + + Database.sequelize = { + getDialect: () => 'postgres', + query: queryStub + } + + const result = await libraryItemsPodcastFilters.getPodcastLibraryStats('library-1') + + expect(queryStub.secondCall.args[0]).to.include('NULLIF(pe.audioFile::jsonb #>>') + expect(result).to.deep.equal({ + totalSize: 1000, + totalDuration: '12.3', + numAudioFiles: '9', + totalItems: '4' + }) + }) +}) diff --git a/test/server/utils/queries/seriesFilters.test.js b/test/server/utils/queries/seriesFilters.test.js new file mode 100644 index 000000000..eb1c3cb54 --- /dev/null +++ b/test/server/utils/queries/seriesFilters.test.js @@ -0,0 +1,107 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Database = require('../../../../server/Database') +const seriesFilters = require('../../../../server/utils/queries/seriesFilters') +const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters') + +function encodedFilter(group, value) { + return `${group}.${encodeURIComponent(Buffer.from(value).toString('base64'))}` +} + +describe('seriesFilters dialect behavior', () => { + let originalSequelize + let originalServerSettings + let modelStubs + + function setDialect(dialect) { + Database.sequelize = { + getDialect: () => dialect, + random: sinon.stub(), + models: modelStubs + } + } + + beforeEach(() => { + originalSequelize = Database.sequelize + originalServerSettings = global.ServerSettings + modelStubs = { + series: { + findAndCountAll: sinon.stub().resolves({ rows: [], count: 0 }) + }, + bookSeries: {}, + book: {}, + libraryItem: {}, + author: {}, + feed: {} + } + + global.ServerSettings = { sortingIgnorePrefix: false } + sinon.stub(libraryItemsBookFilters, 'getUserPermissionBookWhereQuery').returns({ + bookWhere: [], + replacements: {} + }) + }) + + afterEach(() => { + Database.sequelize = originalSequelize + global.ServerSettings = originalServerSettings + sinon.restore() + }) + + it('should build postgres progress filter with TRUE and FALSE literals', async () => { + setDialect('postgres') + + await seriesFilters.getFilteredSeries( + { id: 'library-1', settings: { hideSingleBookSeries: false } }, + { id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } }, + encodedFilter('progress', 'not-started'), + 'name', + false, + [], + 10, + 0 + ) + + const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0] + const progressWhere = findOptions.where[1] + + expect(progressWhere.attribute.val).to.include('mp.isFinished = TRUE') + }) + + it('should build sqlite no-case name sort expression', async () => { + setDialect('sqlite') + + await seriesFilters.getFilteredSeries( + { id: 'library-1', settings: { hideSingleBookSeries: false } }, + { id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } }, + null, + 'name', + false, + [], + 10, + 0 + ) + + const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0] + expect(findOptions.order[0][0].val).to.equal('series.name COLLATE NOCASE') + }) + + it('should build postgres lower-case name sort expression', async () => { + setDialect('postgres') + + await seriesFilters.getFilteredSeries( + { id: 'library-1', settings: { hideSingleBookSeries: false } }, + { id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } }, + null, + 'name', + false, + [], + 10, + 0 + ) + + const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0] + expect(findOptions.order[0][0].val).to.equal('LOWER(series.name)') + }) +}) From 059124ca8cd9f149c6c94ce552f6fefa81c90c70 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Tue, 3 Mar 2026 17:17:01 -0500 Subject: [PATCH 19/38] skip sqlite-only migrations on postgres --- .../v2.15.0-series-column-unique.js | 8 +++- server/migrations/v2.15.1-reindex-nocase.js | 6 +++ server/migrations/v2.17.3-fk-constraints.js | 6 +++ .../v2.19.1-copy-title-to-library-items.js | 12 ++++++ .../v2.19.4-improve-podcast-queries.js | 12 ++++++ .../v2.15.0-series-column-unique.test.js | 17 ++++++++ .../migrations/v2.15.1-reindex-nocase.test.js | 41 +++++++++++++++++++ .../migrations/v2.17.3-fk-constraints.test.js | 10 +++++ ...2.19.1-copy-title-to-library-items.test.js | 12 ++++++ .../v2.19.4-improve-podcast-queries.test.js | 12 ++++++ 10 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 test/server/migrations/v2.15.1-reindex-nocase.test.js diff --git a/server/migrations/v2.15.0-series-column-unique.js b/server/migrations/v2.15.0-series-column-unique.js index 7f8526f97..be782225a 100644 --- a/server/migrations/v2.15.0-series-column-unique.js +++ b/server/migrations/v2.15.0-series-column-unique.js @@ -19,8 +19,12 @@ async function up({ context: { queryInterface, logger } }) { logger.info('[2.15.0 migration] UPGRADE BEGIN: 2.15.0-series-column-unique ') // Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0 - logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') - await queryInterface.sequelize.query('REINDEX NOCASE;') + if (queryInterface.sequelize.getDialect() === 'sqlite') { + logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') + await queryInterface.sequelize.query('REINDEX NOCASE;') + } else { + logger.info('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect') + } // Check if the unique index already exists const seriesIndexes = await queryInterface.showIndex('Series') diff --git a/server/migrations/v2.15.1-reindex-nocase.js b/server/migrations/v2.15.1-reindex-nocase.js index 2ec9487d2..429edf9ff 100644 --- a/server/migrations/v2.15.1-reindex-nocase.js +++ b/server/migrations/v2.15.1-reindex-nocase.js @@ -17,6 +17,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info('[2.15.1 migration] UPGRADE BEGIN: 2.15.1-reindex-nocase ') + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect') + logger.info('[2.15.1 migration] UPGRADE END: 2.15.1-reindex-nocase ') + return + } + // Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0 logger.info('[2.15.1 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') await queryInterface.sequelize.query('REINDEX NOCASE;') diff --git a/server/migrations/v2.17.3-fk-constraints.js b/server/migrations/v2.17.3-fk-constraints.js index 5f8a5c9a6..3ad61e585 100644 --- a/server/migrations/v2.17.3-fk-constraints.js +++ b/server/migrations/v2.17.3-fk-constraints.js @@ -18,6 +18,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info('[2.17.3 migration] UPGRADE BEGIN: 2.17.3-fk-constraints') + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect') + logger.info('[2.17.3 migration] UPGRADE END: 2.17.3-fk-constraints') + return + } + const execQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize) // Disable foreign key constraints for the next sequence of operations diff --git a/server/migrations/v2.19.1-copy-title-to-library-items.js b/server/migrations/v2.19.1-copy-title-to-library-items.js index 7b75fa855..d62b008ed 100644 --- a/server/migrations/v2.19.1-copy-title-to-library-items.js +++ b/server/migrations/v2.19.1-copy-title-to-library-items.js @@ -25,6 +25,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`) + logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`) + return + } + await addColumn(queryInterface, logger, 'libraryItems', 'title', { type: queryInterface.sequelize.Sequelize.STRING, allowNull: true }) await copyColumn(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId') await addTrigger(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId') @@ -51,6 +57,12 @@ async function down({ context: { queryInterface, logger } }) { // Downward migration script logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`) + logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`) + return + } + await removeIndex(queryInterface, logger, 'libraryItems', ['libraryId', 'mediaType', 'title']) await removeTrigger(queryInterface, logger, 'libraryItems', 'title') await removeColumn(queryInterface, logger, 'libraryItems', 'title') diff --git a/server/migrations/v2.19.4-improve-podcast-queries.js b/server/migrations/v2.19.4-improve-podcast-queries.js index 689795c31..3becd3162 100644 --- a/server/migrations/v2.19.4-improve-podcast-queries.js +++ b/server/migrations/v2.19.4-improve-podcast-queries.js @@ -26,6 +26,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`) + logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`) + return + } + // Add numEpisodes column to podcasts table await addColumn(queryInterface, logger, 'podcasts', 'numEpisodes', { type: queryInterface.sequelize.Sequelize.INTEGER, allowNull: false, defaultValue: 0 }) @@ -60,6 +66,12 @@ async function down({ context: { queryInterface, logger } }) { // Downward migration script logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`) + logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`) + return + } + // Remove triggers from libraryItems await removeTrigger(queryInterface, logger, 'podcasts', 'title', 'libraryItems', 'title') await removeTrigger(queryInterface, logger, 'podcasts', 'titleIgnorePrefix', 'libraryItems', 'titleIgnorePrefix') diff --git a/test/server/migrations/v2.15.0-series-column-unique.test.js b/test/server/migrations/v2.15.0-series-column-unique.test.js index 7fce7b7a1..0439dbbb5 100644 --- a/test/server/migrations/v2.15.0-series-column-unique.test.js +++ b/test/server/migrations/v2.15.0-series-column-unique.test.js @@ -88,6 +88,23 @@ describe('migration-v2.15.0-series-column-unique', () => { await queryInterface.dropTable('Series') await queryInterface.dropTable('BookSeries') }) + + it('should skip NOCASE reindex on postgres dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + const originalQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize) + const queryStub = sinon.stub(queryInterface.sequelize, 'query').callsFake((sql, options) => { + if (sql === 'REINDEX NOCASE;') { + throw new Error('Unexpected NOCASE reindex in postgres mode') + } + return originalQuery(sql, options) + }) + + await up({ context: { queryInterface, logger: Logger } }) + + expect(queryStub.neverCalledWith('REINDEX NOCASE;')).to.be.true + expect(loggerInfoStub.calledWith(sinon.match('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.be.true + }) + it('upgrade with no duplicate series', async () => { // Add some entries to the Series table using the UUID for the ids await queryInterface.bulkInsert('Series', [ diff --git a/test/server/migrations/v2.15.1-reindex-nocase.test.js b/test/server/migrations/v2.15.1-reindex-nocase.test.js new file mode 100644 index 000000000..50fe89174 --- /dev/null +++ b/test/server/migrations/v2.15.1-reindex-nocase.test.js @@ -0,0 +1,41 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const { up, down } = require('../../../server/migrations/v2.15.1-reindex-nocase') +const Logger = require('../../../server/Logger') + +describe('migration-v2.15.1-reindex-nocase', () => { + afterEach(() => { + sinon.restore() + }) + + it('should skip reindex on non-sqlite dialect', async () => { + const queryStub = sinon.stub().resolves() + const loggerInfoStub = sinon.stub(Logger, 'info') + const queryInterface = { + sequelize: { + getDialect: () => 'postgres', + query: queryStub + } + } + + await up({ context: { queryInterface, logger: Logger } }) + + expect(queryStub.called).to.equal(false) + expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.equal(true) + }) + + it('should log no-op on down migration', async () => { + const loggerInfoStub = sinon.stub(Logger, 'info') + const queryInterface = { + sequelize: { + getDialect: () => 'postgres', + query: sinon.stub().resolves() + } + } + + await down({ context: { queryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] No action required for downgrade'))).to.equal(true) + }) +}) diff --git a/test/server/migrations/v2.17.3-fk-constraints.test.js b/test/server/migrations/v2.17.3-fk-constraints.test.js index 33be43ce8..975302831 100644 --- a/test/server/migrations/v2.17.3-fk-constraints.test.js +++ b/test/server/migrations/v2.17.3-fk-constraints.test.js @@ -34,6 +34,16 @@ describe('migration-v2.17.3-fk-constraints', () => { await queryInterface.dropAllTables() }) + it('should skip sqlite-specific migration on non-sqlite dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + const queryStub = sinon.stub(queryInterface.sequelize, 'query').resolves([]) + + await up({ context: { queryInterface, logger: Logger } }) + + expect(queryStub.called).to.equal(false) + expect(loggerInfoStub.calledWith(sinon.match('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect'))).to.equal(true) + }) + it('should fix table foreign key constraints', async () => { // Create tables with missing foreign key constraints: libraryItems, feeds, mediaItemShares, playbackSessions, playlistMediaItems, mediaProgresses await queryInterface.sequelize.query('CREATE TABLE `libraryItems` (`id` UUID UNIQUE PRIMARY KEY, `libraryId` UUID REFERENCES `libraries` (`id`), `libraryFolderId` UUID REFERENCES `libraryFolders` (`id`));') diff --git a/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js b/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js index 5b7678567..7e291927d 100644 --- a/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js +++ b/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js @@ -47,6 +47,18 @@ describe('Migration v2.19.1-copy-title-to-library-items', () => { }) describe('up', () => { + it('should skip sqlite-specific migration on non-sqlite dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + + await up({ context: { queryInterface, logger: Logger } }) + + const table = await queryInterface.describeTable('libraryItems') + expect(table).to.have.property('id') + expect(table).to.not.have.property('title') + expect(table).to.not.have.property('titleIgnorePrefix') + expect(loggerInfoStub.calledWith(sinon.match('[2.19.1 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true + }) + it('should copy title and titleIgnorePrefix to libraryItems', async () => { await up({ context: { queryInterface, logger: Logger } }) diff --git a/test/server/migrations/v2.19.4-improve-podcast-queries.test.js b/test/server/migrations/v2.19.4-improve-podcast-queries.test.js index 0ca697d70..e385318ad 100644 --- a/test/server/migrations/v2.19.4-improve-podcast-queries.test.js +++ b/test/server/migrations/v2.19.4-improve-podcast-queries.test.js @@ -75,6 +75,18 @@ describe('Migration v2.19.4-improve-podcast-queries', () => { }) describe('up', () => { + it('should skip sqlite-specific migration on non-sqlite dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + + await up({ context: { queryInterface, logger: Logger } }) + + const podcastsTable = await queryInterface.describeTable('podcasts') + const mediaProgressesTable = await queryInterface.describeTable('mediaProgresses') + expect(podcastsTable).to.not.have.property('numEpisodes') + expect(mediaProgressesTable).to.not.have.property('podcastId') + expect(loggerInfoStub.calledWith(sinon.match('[2.19.4 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true + }) + it('should add numEpisodes column to podcasts', async () => { await up({ context: { queryInterface, logger: Logger } }) From a1490522d6f07689a9cc1861af4bc90adab7fc28 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Tue, 3 Mar 2026 17:34:36 -0500 Subject: [PATCH 20/38] harden migration index handling for postgres replays --- .../v2.15.0-series-column-unique.js | 26 ++++++++--- server/migrations/v2.15.2-index-creation.js | 16 ++++++- .../migrations/v2.15.2-index-creation.test.js | 45 +++++++++++++++++++ 3 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 test/server/migrations/v2.15.2-index-creation.test.js diff --git a/server/migrations/v2.15.0-series-column-unique.js b/server/migrations/v2.15.0-series-column-unique.js index be782225a..883ce09fc 100644 --- a/server/migrations/v2.15.0-series-column-unique.js +++ b/server/migrations/v2.15.0-series-column-unique.js @@ -28,7 +28,12 @@ async function up({ context: { queryInterface, logger } }) { // Check if the unique index already exists const seriesIndexes = await queryInterface.showIndex('Series') - if (seriesIndexes.some((index) => index.name === 'unique_series_name_per_library')) { + if ( + seriesIndexes.some((index) => { + const indexName = index?.name || index?.indexName || '' + return String(indexName).toLowerCase() === 'unique_series_name_per_library' + }) + ) { logger.info('[2.15.0 migration] Unique index on Series.name and Series.libraryId already exists') logger.info('[2.15.0 migration] UPGRADE END: 2.15.0-series-column-unique ') return @@ -185,11 +190,20 @@ async function up({ context: { queryInterface, logger } }) { logger.info(`[2.15.0 migration] Deduplication complete`) // Create a unique index based on the name and library ID for the `Series` table - await queryInterface.addIndex('Series', ['name', 'libraryId'], { - unique: true, - name: 'unique_series_name_per_library' - }) - logger.info('[2.15.0 migration] Added unique index on Series.name and Series.libraryId') + try { + await queryInterface.addIndex('Series', ['name', 'libraryId'], { + unique: true, + name: 'unique_series_name_per_library' + }) + logger.info('[2.15.0 migration] Added unique index on Series.name and Series.libraryId') + } catch (error) { + const alreadyExists = + (error?.name === 'SequelizeDatabaseError' && /already exists/i.test(error?.message || '')) || + error?.original?.code === '42P07' + if (!alreadyExists) throw error + + logger.info('[2.15.0 migration] Unique index on Series.name and Series.libraryId already exists') + } logger.info('[2.15.0 migration] UPGRADE END: 2.15.0-series-column-unique ') } diff --git a/server/migrations/v2.15.2-index-creation.js b/server/migrations/v2.15.2-index-creation.js index f1302dd26..5f976495c 100644 --- a/server/migrations/v2.15.2-index-creation.js +++ b/server/migrations/v2.15.2-index-creation.js @@ -41,7 +41,7 @@ async function up({ context: { queryInterface, logger } }) { // Delete existing podcastEpisode index logger.info('[2.15.2 migration] Deleting existing podcastEpisode index') - await queryInterface.removeIndex('podcastEpisodes', 'podcast_episodes_created_at') + await removeIndexIfExists(queryInterface, 'podcastEpisodes', 'podcast_episodes_created_at', logger) // Create index for podcastEpisode and createdAt logger.info('[2.15.2 migration] Creating index for podcastEpisode and createdAt') @@ -78,7 +78,7 @@ async function down({ context: { queryInterface, logger } }) { // Delete existing podcastEpisode index logger.info('[2.15.2 migration] Deleting existing podcastEpisode index') - await queryInterface.removeIndex('podcastEpisodes', 'podcastEpisode_createdAt_podcastId') + await removeIndexIfExists(queryInterface, 'podcastEpisodes', 'podcastEpisode_createdAt_podcastId', logger) // Create index for podcastEpisode and createdAt logger.info('[2.15.2 migration] Creating original index for podcastEpisode createdAt') @@ -91,3 +91,15 @@ async function down({ context: { queryInterface, logger } }) { } module.exports = { up, down } + +async function removeIndexIfExists(queryInterface, tableName, indexName, logger) { + const indexes = await queryInterface.showIndex(tableName) + const hasIndex = indexes.some((index) => String(index?.name || index?.indexName || '').toLowerCase() === indexName.toLowerCase()) + + if (!hasIndex) { + logger.info(`[2.15.2 migration] Index ${indexName} does not exist, skipping removeIndex`) + return + } + + await queryInterface.removeIndex(tableName, indexName) +} diff --git a/test/server/migrations/v2.15.2-index-creation.test.js b/test/server/migrations/v2.15.2-index-creation.test.js new file mode 100644 index 000000000..c69142fcd --- /dev/null +++ b/test/server/migrations/v2.15.2-index-creation.test.js @@ -0,0 +1,45 @@ +const { expect } = require('chai') +const { Sequelize, DataTypes } = require('sequelize') + +const Logger = require('../../../server/Logger') +const { up, down } = require('../../../server/migrations/v2.15.2-index-creation') + +describe('migration-v2.15.2-index-creation', () => { + let sequelize + let queryInterface + + beforeEach(async () => { + sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + queryInterface = sequelize.getQueryInterface() + + await queryInterface.createTable('bookAuthors', { + id: { type: DataTypes.INTEGER, primaryKey: true }, + authorId: { type: DataTypes.INTEGER } + }) + + await queryInterface.createTable('bookSeries', { + id: { type: DataTypes.INTEGER, primaryKey: true }, + seriesId: { type: DataTypes.INTEGER } + }) + + await queryInterface.createTable('podcastEpisodes', { + id: { type: DataTypes.INTEGER, primaryKey: true }, + createdAt: { type: DataTypes.DATE }, + podcastId: { type: DataTypes.INTEGER } + }) + }) + + it('up should succeed when legacy podcast index is missing', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const indexes = await queryInterface.showIndex('podcastEpisodes') + expect(indexes.some((index) => index.name === 'podcastEpisode_createdAt_podcastId')).to.equal(true) + }) + + it('down should succeed when new podcast index is missing', async () => { + await down({ context: { queryInterface, logger: Logger } }) + + const indexes = await queryInterface.showIndex('podcastEpisodes') + expect(indexes.some((index) => index.name === 'podcast_episodes_created_at')).to.equal(true) + }) +}) From ce2efe6a016eeef544129369b5e5006643a7a0f6 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Tue, 3 Mar 2026 17:42:49 -0500 Subject: [PATCH 21/38] make migration replays resilient on postgres --- server/migrations/v2.15.2-index-creation.js | 40 +++++++----- ....4-use-subfolder-for-oidc-redirect-uris.js | 17 +++-- .../migrations/v2.15.2-index-creation.test.js | 53 ++++++++++++++++ ...e-subfolder-for-oidc-redirect-uris.test.js | 63 ++++++++++++++++--- 4 files changed, 145 insertions(+), 28 deletions(-) diff --git a/server/migrations/v2.15.2-index-creation.js b/server/migrations/v2.15.2-index-creation.js index 5f976495c..03113e8df 100644 --- a/server/migrations/v2.15.2-index-creation.js +++ b/server/migrations/v2.15.2-index-creation.js @@ -20,10 +20,8 @@ async function up({ context: { queryInterface, logger } }) { // Create index for bookAuthors logger.info('[2.15.2 migration] Creating index for bookAuthors') const bookAuthorsIndexes = await queryInterface.showIndex('bookAuthors') - if (!bookAuthorsIndexes.some((index) => index.name === 'bookAuthor_authorId')) { - await queryInterface.addIndex('bookAuthors', ['authorId'], { - name: 'bookAuthor_authorId' - }) + if (!hasIndex(bookAuthorsIndexes, 'bookAuthor_authorId')) { + await addIndexIfMissing(queryInterface, 'bookAuthors', ['authorId'], 'bookAuthor_authorId', logger) } else { logger.info('[2.15.2 migration] Index bookAuthor_authorId already exists') } @@ -31,10 +29,8 @@ async function up({ context: { queryInterface, logger } }) { // Create index for bookSeries logger.info('[2.15.2 migration] Creating index for bookSeries') const bookSeriesIndexes = await queryInterface.showIndex('bookSeries') - if (!bookSeriesIndexes.some((index) => index.name === 'bookSeries_seriesId')) { - await queryInterface.addIndex('bookSeries', ['seriesId'], { - name: 'bookSeries_seriesId' - }) + if (!hasIndex(bookSeriesIndexes, 'bookSeries_seriesId')) { + await addIndexIfMissing(queryInterface, 'bookSeries', ['seriesId'], 'bookSeries_seriesId', logger) } else { logger.info('[2.15.2 migration] Index bookSeries_seriesId already exists') } @@ -46,10 +42,8 @@ async function up({ context: { queryInterface, logger } }) { // Create index for podcastEpisode and createdAt logger.info('[2.15.2 migration] Creating index for podcastEpisode and createdAt') const podcastEpisodesIndexes = await queryInterface.showIndex('podcastEpisodes') - if (!podcastEpisodesIndexes.some((index) => index.name === 'podcastEpisode_createdAt_podcastId')) { - await queryInterface.addIndex('podcastEpisodes', ['createdAt', 'podcastId'], { - name: 'podcastEpisode_createdAt_podcastId' - }) + if (!hasIndex(podcastEpisodesIndexes, 'podcastEpisode_createdAt_podcastId')) { + await addIndexIfMissing(queryInterface, 'podcastEpisodes', ['createdAt', 'podcastId'], 'podcastEpisode_createdAt_podcastId', logger) } else { logger.info('[2.15.2 migration] Index podcastEpisode_createdAt_podcastId already exists') } @@ -94,12 +88,30 @@ module.exports = { up, down } async function removeIndexIfExists(queryInterface, tableName, indexName, logger) { const indexes = await queryInterface.showIndex(tableName) - const hasIndex = indexes.some((index) => String(index?.name || index?.indexName || '').toLowerCase() === indexName.toLowerCase()) + const hasIndexWithName = hasIndex(indexes, indexName) - if (!hasIndex) { + if (!hasIndexWithName) { logger.info(`[2.15.2 migration] Index ${indexName} does not exist, skipping removeIndex`) return } await queryInterface.removeIndex(tableName, indexName) } + +function hasIndex(indexes, indexName) { + const expected = String(indexName || '').toLowerCase() + return indexes.some((index) => String(index?.name || index?.indexName || '').toLowerCase() === expected) +} + +async function addIndexIfMissing(queryInterface, tableName, fields, indexName, logger) { + try { + await queryInterface.addIndex(tableName, fields, { name: indexName }) + } catch (error) { + const alreadyExists = + (error?.name === 'SequelizeDatabaseError' && /already exists/i.test(error?.message || '')) || + error?.original?.code === '42P07' + if (!alreadyExists) throw error + + logger.info(`[2.15.2 migration] Index ${indexName} already exists`) + } +} diff --git a/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.js b/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.js index 03797e35e..08ab3b983 100644 --- a/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.js +++ b/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.js @@ -56,15 +56,23 @@ async function down({ context: { queryInterface, logger } }) { } async function getServerSettings(queryInterface, logger) { - const result = await queryInterface.sequelize.query('SELECT value FROM settings WHERE key = "server-settings";') + const result = await queryInterface.sequelize.query('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) if (!result[0].length) { logger.error('[2.17.4 migration] Server settings not found') throw new Error('Server settings not found') } + const settingsValue = result[0][0].value + + if (settingsValue && typeof settingsValue === 'object') { + return settingsValue + } + let serverSettings = null try { - serverSettings = JSON.parse(result[0][0].value) + serverSettings = JSON.parse(settingsValue) } catch (error) { logger.error('[2.17.4 migration] Error parsing server settings:', error) throw error @@ -74,9 +82,10 @@ async function getServerSettings(queryInterface, logger) { } async function updateServerSettings(queryInterface, logger, serverSettings) { - await queryInterface.sequelize.query('UPDATE settings SET value = :value WHERE key = "server-settings";', { + await queryInterface.sequelize.query('UPDATE settings SET value = :value WHERE key = :settingsKey;', { replacements: { - value: JSON.stringify(serverSettings) + value: JSON.stringify(serverSettings), + settingsKey: 'server-settings' } }) } diff --git a/test/server/migrations/v2.15.2-index-creation.test.js b/test/server/migrations/v2.15.2-index-creation.test.js index c69142fcd..d9257a483 100644 --- a/test/server/migrations/v2.15.2-index-creation.test.js +++ b/test/server/migrations/v2.15.2-index-creation.test.js @@ -1,5 +1,6 @@ const { expect } = require('chai') const { Sequelize, DataTypes } = require('sequelize') +const sinon = require('sinon') const Logger = require('../../../server/Logger') const { up, down } = require('../../../server/migrations/v2.15.2-index-creation') @@ -29,6 +30,10 @@ describe('migration-v2.15.2-index-creation', () => { }) }) + afterEach(async () => { + if (sequelize) await sequelize.close() + }) + it('up should succeed when legacy podcast index is missing', async () => { await up({ context: { queryInterface, logger: Logger } }) @@ -42,4 +47,52 @@ describe('migration-v2.15.2-index-creation', () => { const indexes = await queryInterface.showIndex('podcastEpisodes') expect(indexes.some((index) => index.name === 'podcast_episodes_created_at')).to.equal(true) }) + + it('up should treat index names case-insensitively', async () => { + const qi = { + showIndex: sinon.stub(), + addIndex: sinon.stub().resolves(), + removeIndex: sinon.stub().resolves() + } + + qi.showIndex.onCall(0).resolves([{ name: 'bookauthor_authorid' }]) + qi.showIndex.onCall(1).resolves([{ name: 'bookseries_seriesid' }]) + qi.showIndex.onCall(2).resolves([{ name: 'podcast_episodes_created_at' }]) + qi.showIndex.onCall(3).resolves([{ name: 'podcastepisode_createdat_podcastid' }]) + + await up({ context: { queryInterface: qi, logger: Logger } }) + + expect(qi.addIndex.called).to.equal(false) + expect(qi.removeIndex.calledOnceWithExactly('podcastEpisodes', 'podcast_episodes_created_at')).to.equal(true) + }) + + it('up should continue when addIndex reports already exists', async () => { + const makePgExistsError = (sql) => { + const err = new Error('relation already exists') + err.name = 'SequelizeDatabaseError' + err.original = { code: '42P07' } + err.sql = sql + return err + } + + const qi = { + showIndex: sinon.stub(), + addIndex: sinon.stub(), + removeIndex: sinon.stub().resolves() + } + + qi.showIndex.onCall(0).resolves([]) + qi.showIndex.onCall(1).resolves([]) + qi.showIndex.onCall(2).resolves([{ name: 'podcast_episodes_created_at' }]) + qi.showIndex.onCall(3).resolves([]) + + qi.addIndex.onCall(0).rejects(makePgExistsError('CREATE INDEX bookAuthor_authorId ON bookAuthors (authorId)')) + qi.addIndex.onCall(1).rejects(makePgExistsError('CREATE INDEX bookSeries_seriesId ON bookSeries (seriesId)')) + qi.addIndex.onCall(2).rejects(makePgExistsError('CREATE INDEX podcastEpisode_createdAt_podcastId ON podcastEpisodes (createdAt, podcastId)')) + + await up({ context: { queryInterface: qi, logger: Logger } }) + + expect(qi.addIndex.callCount).to.equal(3) + expect(qi.removeIndex.calledOnceWithExactly('podcastEpisodes', 'podcast_episodes_created_at')).to.equal(true) + }) }) diff --git a/test/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.test.js b/test/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.test.js index 1662d5f98..6794a5696 100644 --- a/test/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.test.js +++ b/test/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.test.js @@ -30,11 +30,16 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { expect(logger.info.calledWith('[2.17.4 migration] UPGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] OIDC is enabled, adding authOpenIDSubfolderForRedirectURLs to server settings')).to.be.true expect(queryInterface.sequelize.query.calledTwice).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true expect( - queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = "server-settings";', { + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true + expect( + queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', { replacements: { - value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }) + value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }), + settingsKey: 'server-settings' } }) ).to.be.true @@ -49,10 +54,31 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { expect(logger.info.calledWith('[2.17.4 migration] UPGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] OIDC is not enabled, no action required')).to.be.true expect(queryInterface.sequelize.query.calledOnce).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true + expect( + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true expect(logger.info.calledWith('[2.17.4 migration] UPGRADE END: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true }) + it('should handle already-parsed object server settings', async () => { + queryInterface.sequelize.query.onFirstCall().resolves([[{ value: { authActiveAuthMethods: ['openid'] } }]]) + queryInterface.sequelize.query.onSecondCall().resolves() + + await up({ context }) + + expect(queryInterface.sequelize.query.calledTwice).to.be.true + expect( + queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', { + replacements: { + value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }), + settingsKey: 'server-settings' + } + }) + ).to.be.true + }) + it('should throw an error if server settings cannot be parsed', async () => { queryInterface.sequelize.query.onFirstCall().resolves([[{ value: 'invalid json' }]]) @@ -60,7 +86,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { await up({ context }) } catch (error) { expect(queryInterface.sequelize.query.calledOnce).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true + expect( + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true expect(logger.error.calledWith('[2.17.4 migration] Error parsing server settings:')).to.be.true expect(error).to.be.instanceOf(Error) } @@ -73,7 +103,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { await up({ context }) } catch (error) { expect(queryInterface.sequelize.query.calledOnce).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true + expect( + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true expect(logger.error.calledWith('[2.17.4 migration] Server settings not found')).to.be.true expect(error).to.be.instanceOf(Error) } @@ -90,11 +124,16 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] Removing authOpenIDSubfolderForRedirectURLs from server settings')).to.be.true expect(queryInterface.sequelize.query.calledTwice).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true expect( - queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = "server-settings";', { + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true + expect( + queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', { replacements: { - value: JSON.stringify({}) + value: JSON.stringify({}), + settingsKey: 'server-settings' } }) ).to.be.true @@ -109,7 +148,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => { expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] authOpenIDSubfolderForRedirectURLs not found in server settings, no action required')).to.be.true expect(queryInterface.sequelize.query.calledOnce).to.be.true - expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true + expect( + queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', { + replacements: { settingsKey: 'server-settings' } + }) + ).to.be.true expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE END: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true }) }) From 6183b3012cf94cf1ae66cb0ef1bf0336aec33a62 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Tue, 3 Mar 2026 21:37:52 -0500 Subject: [PATCH 22/38] prevent discover query crashes from restarting server --- server/utils/queries/libraryFilters.js | 31 +++++++------ .../utils/queries/libraryItemsBookFilters.js | 2 +- .../utils/queries/libraryFilters.test.js | 45 +++++++++++++++++++ .../queries/libraryItemsBookFilters.test.js | 15 +++++++ 4 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 test/server/utils/queries/libraryFilters.test.js diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index 49f5381e1..ada2965e5 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -374,19 +374,24 @@ module.exports = { async getLibraryItemsToDiscover(library, user, include, limit) { if (library.mediaType !== 'book') return { libraryItems: [], count: 0 } - const { libraryItems, count } = await libraryItemsBookFilters.getDiscoverLibraryItems(library.id, user, include, limit) - return { - libraryItems: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - if (li.rssFeed) { - oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() - } - if (li.mediaItemShare) { - oldLibraryItem.mediaItemShare = li.mediaItemShare - } - return oldLibraryItem - }), - count + try { + const { libraryItems, count } = await libraryItemsBookFilters.getDiscoverLibraryItems(library.id, user, include, limit) + return { + libraryItems: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + if (li.rssFeed) { + oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() + } + if (li.mediaItemShare) { + oldLibraryItem.mediaItemShare = li.mediaItemShare + } + return oldLibraryItem + }), + count + } + } catch (error) { + Logger.error(`[LibraryFilters] Failed to load discover shelf for library "${library.id}"`, error) + return { libraryItems: [], count: 0 } } }, diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 0a5a6a4ed..18f0918ad 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -942,7 +942,7 @@ module.exports = { const discoverWhere = [ { '$mediaProgresses.isFinished$': { - [Sequelize.Op.or]: [null, 0] + [Sequelize.Op.or]: [null, false] }, '$mediaProgresses.currentTime$': { [Sequelize.Op.or]: [null, 0] diff --git a/test/server/utils/queries/libraryFilters.test.js b/test/server/utils/queries/libraryFilters.test.js new file mode 100644 index 000000000..c88aa63ed --- /dev/null +++ b/test/server/utils/queries/libraryFilters.test.js @@ -0,0 +1,45 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Logger = require('../../../../server/Logger') +const libraryFilters = require('../../../../server/utils/queries/libraryFilters') +const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters') + +describe('libraryFilters discover shelf resilience', () => { + afterEach(() => { + sinon.restore() + }) + + it('should return empty discover shelf when discover query fails', async () => { + sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').rejects(new Error('discover failed')) + const errorStub = sinon.stub(Logger, 'error') + + const result = await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10) + + expect(result).to.deep.equal({ libraryItems: [], count: 0 }) + expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load discover shelf for library "library-1"')).to.equal(true) + }) + + it('should keep discover shelf mapping behavior when query succeeds', async () => { + const libraryItem = { + toOldJSONMinified: () => ({ id: 'item-1' }), + rssFeed: { toOldJSONMinified: () => ({ id: 'rss-1' }) }, + mediaItemShare: { id: 'share-1' } + } + sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').resolves({ + libraryItems: [libraryItem], + count: 1 + }) + + const result = await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10) + + expect(result.count).to.equal(1) + expect(result.libraryItems).to.deep.equal([ + { + id: 'item-1', + rssFeed: { id: 'rss-1' }, + mediaItemShare: { id: 'share-1' } + } + ]) + }) +}) diff --git a/test/server/utils/queries/libraryItemsBookFilters.test.js b/test/server/utils/queries/libraryItemsBookFilters.test.js index 19b6a58bc..9be7baff4 100644 --- a/test/server/utils/queries/libraryItemsBookFilters.test.js +++ b/test/server/utils/queries/libraryItemsBookFilters.test.js @@ -1,5 +1,6 @@ const { expect } = require('chai') const sinon = require('sinon') +const Sequelize = require('sequelize') const Database = require('../../../../server/Database') const Logger = require('../../../../server/Logger') @@ -130,4 +131,18 @@ describe('libraryItemsBookFilters postgres query safety', () => { expect(orderExpression).to.include('"books->bookSeries"."sequence"') }) + + it('should use boolean false for discover not-started media progress on postgres', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.series.findAll.resolves([]) + modelStubs.book.count.resolves(0) + modelStubs.book.findAll.resolves([]) + + await libraryItemsBookFilters.getDiscoverLibraryItems('library-1', { id: 'user-1', canAccessExplicitContent: true }, [], 10) + + const countOptions = modelStubs.book.count.firstCall.args[0] + const progressFilter = countOptions.where[0]['$mediaProgresses.isFinished$'] + + expect(progressFilter[Sequelize.Op.or]).to.deep.equal([null, false]) + }) }) From 34855e00fcdbb417ab9669c876920c4bf994e1d1 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sat, 7 Mar 2026 17:35:05 -0500 Subject: [PATCH 23/38] harden shelf queries and guard boolean sql --- server/utils/queries/libraryFilters.js | 324 ++++++++++-------- .../utils/queries/libraryFilters.test.js | 33 +- .../utils/queries/sqlBooleanSafety.test.js | 47 +++ 3 files changed, 251 insertions(+), 153 deletions(-) create mode 100644 test/server/utils/queries/sqlBooleanSafety.test.js diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index ada2965e5..fd261e050 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -10,6 +10,15 @@ const naturalSort = createNewSortInstance({ comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare }) +async function withShelfFallback(scope, fallbackValue, action) { + try { + return await action() + } catch (error) { + Logger.error(`[LibraryFilters] Failed to load ${scope}`, error) + return fallbackValue + } +} + module.exports = { decode(text) { return Buffer.from(decodeURIComponent(text), 'base64').toString() @@ -50,22 +59,24 @@ module.exports = { * @returns {Promise<{ items:import('../../models/LibraryItem')[], count:number }>} */ async getMediaItemsInProgress(library, user, include, limit) { - if (library.isBook) { - const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'in-progress', 'progress', true, false, include, limit, 0, true) - return { - items: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - if (li.rssFeed) { - oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() - } - if (li.mediaItemShare) { - oldLibraryItem.mediaItemShare = li.mediaItemShare - } - return oldLibraryItem - }), - count + return withShelfFallback(`in-progress shelf for library "${library.id}"`, { items: [], count: 0 }, async () => { + if (library.isBook) { + const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'in-progress', 'progress', true, false, include, limit, 0, true) + return { + items: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + if (li.rssFeed) { + oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() + } + if (li.mediaItemShare) { + oldLibraryItem.mediaItemShare = li.mediaItemShare + } + return oldLibraryItem + }), + count + } } - } else { + const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'in-progress', 'progress', true, limit, 0, true) return { count, @@ -75,7 +86,7 @@ module.exports = { return oldLibraryItem }) } - } + }) }, /** @@ -87,25 +98,27 @@ module.exports = { * @returns {object} { libraryItems:LibraryItem[], count:number } */ async getLibraryItemsMostRecentlyAdded(library, user, include, limit) { - if (library.isBook) { - const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, false, include, limit, 0) - return { - libraryItems: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - if (li.rssFeed) { - oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() - } - if (li.size && !oldLibraryItem.media.size) { - oldLibraryItem.media.size = li.size - } - if (li.mediaItemShare) { - oldLibraryItem.mediaItemShare = li.mediaItemShare - } - return oldLibraryItem - }), - count + return withShelfFallback(`recent shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => { + if (library.isBook) { + const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, false, include, limit, 0) + return { + libraryItems: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + if (li.rssFeed) { + oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() + } + if (li.size && !oldLibraryItem.media.size) { + oldLibraryItem.media.size = li.size + } + if (li.mediaItemShare) { + oldLibraryItem.mediaItemShare = li.mediaItemShare + } + return oldLibraryItem + }), + count + } } - } else { + const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, include, limit, 0) return { libraryItems: libraryItems.map((li) => { @@ -123,7 +136,7 @@ module.exports = { }), count } - } + }) }, /** @@ -135,23 +148,25 @@ module.exports = { * @returns {object} { libraryItems:LibraryItem[], count:number } */ async getLibraryItemsContinueSeries(library, user, include, limit) { - const { libraryItems, count } = await libraryItemsBookFilters.getContinueSeriesLibraryItems(library, user, include, limit, 0) - return { - libraryItems: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - if (li.rssFeed) { - oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() - } - if (li.series) { - oldLibraryItem.media.metadata.series = li.series - } - if (li.mediaItemShare) { - oldLibraryItem.mediaItemShare = li.mediaItemShare - } - return oldLibraryItem - }), - count - } + return withShelfFallback(`continue-series shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => { + const { libraryItems, count } = await libraryItemsBookFilters.getContinueSeriesLibraryItems(library, user, include, limit, 0) + return { + libraryItems: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + if (li.rssFeed) { + oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() + } + if (li.series) { + oldLibraryItem.media.metadata.series = li.series + } + if (li.mediaItemShare) { + oldLibraryItem.mediaItemShare = li.mediaItemShare + } + return oldLibraryItem + }), + count + } + }) }, /** @@ -164,22 +179,24 @@ module.exports = { * @returns {Promise<{ items:oldLibraryItem[], count:number }>} */ async getMediaFinished(library, user, include, limit) { - if (library.isBook) { - const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'finished', 'progress', true, false, include, limit, 0) - return { - items: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - if (li.rssFeed) { - oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() - } - if (li.mediaItemShare) { - oldLibraryItem.mediaItemShare = li.mediaItemShare - } - return oldLibraryItem - }), - count + return withShelfFallback(`finished shelf for library "${library.id}"`, { items: [], count: 0 }, async () => { + if (library.isBook) { + const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'finished', 'progress', true, false, include, limit, 0) + return { + items: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + if (li.rssFeed) { + oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() + } + if (li.mediaItemShare) { + oldLibraryItem.mediaItemShare = li.mediaItemShare + } + return oldLibraryItem + }), + count + } } - } else { + const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'finished', 'progress', true, limit, 0) return { count, @@ -189,7 +206,7 @@ module.exports = { return oldLibraryItem }) } - } + }) }, /** @@ -203,23 +220,24 @@ module.exports = { async getSeriesMostRecentlyAdded(library, user, include, limit) { if (!library.isBook) return { series: [], count: 0 } - const seriesIncludes = [] - if (include.includes('rssfeed')) { - seriesIncludes.push({ - model: Database.feedModel - }) - } - - const userPermissionBookWhere = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user) - - const seriesWhere = [ - { - libraryId: library.id, - createdAt: { - [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago - } + return withShelfFallback(`recent-series shelf for library "${library.id}"`, { series: [], count: 0 }, async () => { + const seriesIncludes = [] + if (include.includes('rssfeed')) { + seriesIncludes.push({ + model: Database.feedModel + }) } - ] + + const userPermissionBookWhere = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user) + + const seriesWhere = [ + { + libraryId: library.id, + createdAt: { + [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago + } + } + ] // Handle library setting to hide single book series // TODO: Merge with existing query @@ -252,29 +270,29 @@ module.exports = { ) } - const { rows: series, count } = await Database.seriesModel.findAndCountAll({ - where: seriesWhere, - limit, - offset: 0, - distinct: true, - subQuery: false, - replacements: userPermissionBookWhere.replacements, - include: [ - { - model: Database.bookSeriesModel, - include: { - model: Database.bookModel, - where: userPermissionBookWhere.bookWhere, + const { rows: series, count } = await Database.seriesModel.findAndCountAll({ + where: seriesWhere, + limit, + offset: 0, + distinct: true, + subQuery: false, + replacements: userPermissionBookWhere.replacements, + include: [ + { + model: Database.bookSeriesModel, include: { - model: Database.libraryItemModel - } + model: Database.bookModel, + where: userPermissionBookWhere.bookWhere, + include: { + model: Database.libraryItemModel + } + }, + separate: true }, - separate: true - }, - ...seriesIncludes - ], - order: [['createdAt', 'DESC']] - }) + ...seriesIncludes + ], + order: [['createdAt', 'DESC']] + }) const allOldSeries = [] for (const s of series) { @@ -312,10 +330,11 @@ module.exports = { allOldSeries.push(oldSeries) } - return { - series: allOldSeries, - count - } + return { + series: allOldSeries, + count + } + }) }, /** @@ -330,37 +349,39 @@ module.exports = { async getNewestAuthors(library, user, limit) { if (library.mediaType !== 'book') return { authors: [], count: 0 } - const { bookWhere, replacements } = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user) + return withShelfFallback(`newest-authors shelf for library "${library.id}"`, { authors: [], count: 0 }, async () => { + const { bookWhere, replacements } = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user) - const { rows: authors, count } = await Database.authorModel.findAndCountAll({ - where: { - libraryId: library.id, - createdAt: { - [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago - } - }, - replacements, - include: { - model: Database.bookModel, - attributes: ['id', 'tags', 'explicit'], - where: bookWhere, - required: true, // Must belong to a book - through: { - attributes: [] - } - }, - limit, - distinct: true, - order: [['createdAt', 'DESC']] + const { rows: authors, count } = await Database.authorModel.findAndCountAll({ + where: { + libraryId: library.id, + createdAt: { + [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago + } + }, + replacements, + include: { + model: Database.bookModel, + attributes: ['id', 'tags', 'explicit'], + where: bookWhere, + required: true, // Must belong to a book + through: { + attributes: [] + } + }, + limit, + distinct: true, + order: [['createdAt', 'DESC']] + }) + + return { + authors: authors.map((au) => { + const numBooks = au.books.length || 0 + return au.toOldJSONExpanded(numBooks) + }), + count + } }) - - return { - authors: authors.map((au) => { - const numBooks = au.books.length || 0 - return au.toOldJSONExpanded(numBooks) - }), - count - } }, /** @@ -374,7 +395,7 @@ module.exports = { async getLibraryItemsToDiscover(library, user, include, limit) { if (library.mediaType !== 'book') return { libraryItems: [], count: 0 } - try { + return withShelfFallback(`discover shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => { const { libraryItems, count } = await libraryItemsBookFilters.getDiscoverLibraryItems(library.id, user, include, limit) return { libraryItems: libraryItems.map((li) => { @@ -389,10 +410,7 @@ module.exports = { }), count } - } catch (error) { - Logger.error(`[LibraryFilters] Failed to load discover shelf for library "${library.id}"`, error) - return { libraryItems: [], count: 0 } - } + }) }, /** @@ -405,15 +423,17 @@ module.exports = { async getNewestPodcastEpisodes(library, user, limit) { if (library.mediaType !== 'podcast') return { libraryItems: [], count: 0 } - const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'recent', null, 'createdAt', true, limit, 0) - return { - count, - libraryItems: libraryItems.map((li) => { - const oldLibraryItem = li.toOldJSONMinified() - oldLibraryItem.recentEpisode = li.recentEpisode - return oldLibraryItem - }) - } + return withShelfFallback(`newest-podcast-episodes shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => { + const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'recent', null, 'createdAt', true, limit, 0) + return { + count, + libraryItems: libraryItems.map((li) => { + const oldLibraryItem = li.toOldJSONMinified() + oldLibraryItem.recentEpisode = li.recentEpisode + return oldLibraryItem + }) + } + }) }, /** diff --git a/test/server/utils/queries/libraryFilters.test.js b/test/server/utils/queries/libraryFilters.test.js index c88aa63ed..1c9a2836f 100644 --- a/test/server/utils/queries/libraryFilters.test.js +++ b/test/server/utils/queries/libraryFilters.test.js @@ -4,8 +4,9 @@ const sinon = require('sinon') const Logger = require('../../../../server/Logger') const libraryFilters = require('../../../../server/utils/queries/libraryFilters') const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters') +const libraryItemsPodcastFilters = require('../../../../server/utils/queries/libraryItemsPodcastFilters') -describe('libraryFilters discover shelf resilience', () => { +describe('libraryFilters shelf resilience', () => { afterEach(() => { sinon.restore() }) @@ -20,6 +21,36 @@ describe('libraryFilters discover shelf resilience', () => { expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load discover shelf for library "library-1"')).to.equal(true) }) + it('should return empty in-progress shelf when a query fails', async () => { + sinon.stub(libraryItemsBookFilters, 'getFilteredLibraryItems').rejects(new Error('progress failed')) + const errorStub = sinon.stub(Logger, 'error') + + const result = await libraryFilters.getMediaItemsInProgress({ isBook: true, id: 'library-1' }, { id: 'user-1' }, [], 10) + + expect(result).to.deep.equal({ items: [], count: 0 }) + expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load in-progress shelf for library "library-1"')).to.equal(true) + }) + + it('should return empty continue-series shelf when a query fails', async () => { + sinon.stub(libraryItemsBookFilters, 'getContinueSeriesLibraryItems').rejects(new Error('continue failed')) + const errorStub = sinon.stub(Logger, 'error') + + const result = await libraryFilters.getLibraryItemsContinueSeries({ id: 'library-1' }, { id: 'user-1' }, [], 10) + + expect(result).to.deep.equal({ libraryItems: [], count: 0 }) + expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load continue-series shelf for library "library-1"')).to.equal(true) + }) + + it('should return empty newest podcast episodes shelf when a query fails', async () => { + sinon.stub(libraryItemsPodcastFilters, 'getFilteredPodcastEpisodes').rejects(new Error('podcast failed')) + const errorStub = sinon.stub(Logger, 'error') + + const result = await libraryFilters.getNewestPodcastEpisodes({ mediaType: 'podcast', id: 'library-1' }, { id: 'user-1' }, 10) + + expect(result).to.deep.equal({ libraryItems: [], count: 0 }) + expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load newest-podcast-episodes shelf for library "library-1"')).to.equal(true) + }) + it('should keep discover shelf mapping behavior when query succeeds', async () => { const libraryItem = { toOldJSONMinified: () => ({ id: 'item-1' }), diff --git a/test/server/utils/queries/sqlBooleanSafety.test.js b/test/server/utils/queries/sqlBooleanSafety.test.js new file mode 100644 index 000000000..c70704682 --- /dev/null +++ b/test/server/utils/queries/sqlBooleanSafety.test.js @@ -0,0 +1,47 @@ +const { expect } = require('chai') +const fs = require('fs') +const path = require('path') + +function getJsFiles(dirPath) { + const entries = fs.readdirSync(dirPath, { withFileTypes: true }) + return entries.flatMap((entry) => { + const entryPath = path.join(dirPath, entry.name) + if (entry.isDirectory()) return getJsFiles(entryPath) + if (entry.isFile() && entry.name.endsWith('.js')) return [entryPath] + return [] + }) +} + +describe('query boolean safety', () => { + it('should not use numeric literals for SQL boolean fields in query code', () => { + const queriesDir = path.resolve(__dirname, '../../../../server/utils/queries') + const files = getJsFiles(queriesDir) + const offenders = [] + const forbiddenPatterns = [ + { + regex: /\$mediaProgresses\.isFinished\$[\s\S]{0,120}\[null,\s*[01]\]/g, + message: 'numeric literal in mediaProgresses.isFinished filter' + }, + { + regex: /\$books\.mediaProgresses\.isFinished\$[\s\S]{0,120}\[null,\s*[01]\]/g, + message: 'numeric literal in books.mediaProgresses.isFinished filter' + }, + { + regex: /\b(?:mp|mediaProgresses|b)\.(?:isFinished|explicit|abridged)\s*=\s*[01]\b/g, + message: 'direct SQL boolean comparison against 0/1' + } + ] + + files.forEach((filePath) => { + const content = fs.readFileSync(filePath, 'utf8') + forbiddenPatterns.forEach(({ regex, message }) => { + const matches = [...content.matchAll(regex)] + matches.forEach((match) => { + offenders.push(`${path.relative(queriesDir, filePath)}: ${message}: ${match[0].replace(/\s+/g, ' ').trim()}`) + }) + }) + }) + + expect(offenders).to.deep.equal([]) + }) +}) From 3faf8809e0f3928ae0b2ab4c362dbecd110436c6 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sat, 7 Mar 2026 18:35:02 -0500 Subject: [PATCH 24/38] Improve listening stats query efficiency --- server/Database.js | 14 ++++++- server/controllers/MeController.js | 48 ++++++++++++++---------- server/controllers/UserController.js | 31 ++++++++++------ server/models/PlaybackSession.js | 48 +++++++++++++++++++++--- server/routers/ApiRouter.js | 55 ++++++++++++++++++++++++---- 5 files changed, 151 insertions(+), 45 deletions(-) diff --git a/server/Database.js b/server/Database.js index 3cbf32819..247c7e974 100644 --- a/server/Database.js +++ b/server/Database.js @@ -512,9 +512,19 @@ class Database { return this.models.setting.updateSettingObj(settings.toJSON()) } - getPlaybackSessions(where = null) { + getPlaybackSessions(where = null, options = null) { if (!this.sequelize) return false - return this.models.playbackSession.getOldPlaybackSessions(where) + return this.models.playbackSession.getOldPlaybackSessions(where, options || undefined) + } + + countPlaybackSessions(where = null) { + if (!this.sequelize) return false + return this.models.playbackSession.countWithWhere(where) + } + + getPlaybackSessionsForStats(where = null) { + if (!this.sequelize) return false + return this.models.playbackSession.getPlaybackSessionsForStats(where) } getPlaybackSession(sessionId) { diff --git a/server/controllers/MeController.js b/server/controllers/MeController.js index ca600d373..0365a3105 100644 --- a/server/controllers/MeController.js +++ b/server/controllers/MeController.js @@ -155,21 +155,23 @@ class MeController { * @param {Response} res */ async getListeningSessions(req, res) { - const listeningSessions = await this.getUserListeningSessionsHelper(req.user.id) - + const startedAt = Date.now() const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10 const page = toNumber(req.query.page, 0) - const start = page * itemsPerPage - const sessions = listeningSessions.slice(start, start + itemsPerPage) + Logger.debug( + `[MeController] /api/me/listening-sessions user="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} start` + ) - const payload = { - total: listeningSessions.length, - numPages: Math.ceil(listeningSessions.length / itemsPerPage), + const payload = await this.getUserListeningSessionsPageHelper( + req.user.id, page, - itemsPerPage, - sessions - } + itemsPerPage + ) + + Logger.debug( + `[MeController] /api/me/listening-sessions user="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms` + ) res.json(payload) } @@ -183,6 +185,7 @@ class MeController { * @param {Response} res */ async getItemListeningSessions(req, res) { + const startedAt = Date.now() const libraryItem = await Database.libraryItemModel.getExpandedById(req.params.libraryItemId) const episode = await Database.podcastEpisodeModel.findByPk(req.params.episodeId) @@ -198,21 +201,23 @@ class MeController { } const mediaItemId = episode?.id || libraryItem.mediaId - let listeningSessions = await this.getUserItemListeningSessionsHelper(req.user.id, mediaItemId) - const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10 const page = toNumber(req.query.page, 0) - const start = page * itemsPerPage - const sessions = listeningSessions.slice(start, start + itemsPerPage) + Logger.debug( + `[MeController] /api/me/item/listening-sessions user="${req.user.id}" libraryItem="${req.params.libraryItemId}" episode="${req.params.episodeId || ''}" page=${page} itemsPerPage=${itemsPerPage} start` + ) - const payload = { - total: listeningSessions.length, - numPages: Math.ceil(listeningSessions.length / itemsPerPage), + const payload = await this.getUserListeningSessionsPageHelper( + req.user.id, page, itemsPerPage, - sessions - } + mediaItemId + ) + + Logger.debug( + `[MeController] /api/me/item/listening-sessions user="${req.user.id}" libraryItem="${req.params.libraryItemId}" episode="${req.params.episodeId || ''}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms` + ) res.json(payload) } @@ -226,7 +231,12 @@ class MeController { * @param {Response} res */ async getListeningStats(req, res) { + const startedAt = Date.now() + Logger.debug(`[MeController] /api/me/listening-stats user="${req.user.id}" start`) const listeningStats = await this.getUserListeningStatsHelpers(req.user.id) + Logger.debug( + `[MeController] /api/me/listening-stats user="${req.user.id}" totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms` + ) res.json(listeningStats) } diff --git a/server/controllers/UserController.js b/server/controllers/UserController.js index 0a476a9d6..89ceb56b3 100644 --- a/server/controllers/UserController.js +++ b/server/controllers/UserController.js @@ -432,14 +432,20 @@ class UserController { * @param {Response} res */ async getListeningSessions(req, res) { - var listeningSessions = await this.getUserListeningSessionsHelper(req.params.id) - + const startedAt = Date.now() const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10 const page = toNumber(req.query.page, 0) + Logger.debug( + `[UserController] /api/users/${req.params.id}/listening-sessions reqUser="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} start` + ) + const payload = await this.getUserListeningSessionsPageHelper( + req.params.id, + page, + itemsPerPage + ) - const start = page * itemsPerPage // Map user to sessions to match the format of the sessions endpoint - const sessions = listeningSessions.slice(start, start + itemsPerPage).map((session) => { + payload.sessions = payload.sessions.map((session) => { return { ...session, user: { @@ -449,13 +455,9 @@ class UserController { } }) - const payload = { - total: listeningSessions.length, - numPages: Math.ceil(listeningSessions.length / itemsPerPage), - page, - itemsPerPage, - sessions - } + Logger.debug( + `[UserController] /api/users/${req.params.id}/listening-sessions reqUser="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms` + ) res.json(payload) } @@ -469,7 +471,14 @@ class UserController { * @param {Response} res */ async getListeningStats(req, res) { + const startedAt = Date.now() + Logger.debug( + `[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" start` + ) var listeningStats = await this.getUserListeningStatsHelpers(req.params.id) + Logger.debug( + `[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms` + ) res.json(listeningStats) } diff --git a/server/models/PlaybackSession.js b/server/models/PlaybackSession.js index 47526d5f6..14303a036 100644 --- a/server/models/PlaybackSession.js +++ b/server/models/PlaybackSession.js @@ -52,18 +52,54 @@ class PlaybackSession extends Model { this.createdAt } - static async getOldPlaybackSessions(where = null) { + static async getOldPlaybackSessions(where = null, options = {}) { + const { + limit = null, + offset = null, + order = [['updatedAt', 'DESC']], + includeDevice = true + } = options + const playbackSessions = await this.findAll({ where, - include: [ - { - model: this.sequelize.models.device - } - ] + limit, + offset, + order, + include: includeDevice + ? [ + { + model: this.sequelize.models.device + } + ] + : undefined }) return playbackSessions.map((session) => this.getOldPlaybackSession(session)) } + static countWithWhere(where = null) { + return this.count({ where }) + } + + static getPlaybackSessionsForStats(where = null) { + return this.findAll({ + where, + order: [['updatedAt', 'DESC']], + attributes: [ + 'mediaItemId', + 'mediaItemType', + 'displayTitle', + 'displayAuthor', + 'timeListening', + 'mediaMetadata', + 'date', + 'dayOfWeek', + 'updatedAt', + 'extraData' + ], + raw: true + }) + } + static async getById(sessionId) { const playbackSession = await this.findByPk(sessionId, { include: [ diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index 7c89126b2..fcb414816 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -527,24 +527,57 @@ class ApiRouter { return userSessions.sort((a, b) => b.updatedAt - a.updatedAt) } + async getUserListeningSessionsPageHelper(userId, page = 0, itemsPerPage = 10, mediaItemId = null) { + const where = { userId } + if (mediaItemId) where.mediaItemId = mediaItemId + + const start = page * itemsPerPage + const [total, sessions] = await Promise.all([ + Database.countPlaybackSessions(where), + Database.getPlaybackSessions(where, { + limit: itemsPerPage, + offset: start, + order: [['updatedAt', 'DESC']] + }) + ]) + + return { + total, + numPages: Math.ceil(total / itemsPerPage), + page, + itemsPerPage, + sessions + } + } + async getUserItemListeningSessionsHelper(userId, mediaItemId) { const userSessions = await Database.getPlaybackSessions({ userId, mediaItemId }) return userSessions.sort((a, b) => b.updatedAt - a.updatedAt) } async getUserListeningStatsHelpers(userId) { + const startedAt = Date.now() const today = date.format(new Date(), 'YYYY-MM-DD') - const listeningSessions = await this.getUserListeningSessionsHelper(userId) + const [listeningSessions, recentSessions] = await Promise.all([ + Database.getPlaybackSessionsForStats({ userId }), + Database.getPlaybackSessions({ userId }, { + limit: 10, + offset: 0, + order: [['updatedAt', 'DESC']] + }) + ]) + const listeningStats = { totalTime: 0, items: {}, days: {}, dayOfWeek: {}, today: 0, - recentSessions: listeningSessions.slice(0, 10) + recentSessions } listeningSessions.forEach((s) => { + const libraryItemId = s.extraData?.libraryItemId || null let sessionTimeListening = s.timeListening if (typeof sessionTimeListening == 'string') { sessionTimeListening = Number(sessionTimeListening) @@ -562,19 +595,27 @@ class ApiRouter { listeningStats.today += sessionTimeListening } } - if (!listeningStats.items[s.libraryItemId]) { - listeningStats.items[s.libraryItemId] = { - id: s.libraryItemId, + if (!libraryItemId) { + listeningStats.totalTime += sessionTimeListening + return + } + + if (!listeningStats.items[libraryItemId]) { + listeningStats.items[libraryItemId] = { + id: libraryItemId, timeListening: sessionTimeListening, mediaMetadata: s.mediaMetadata, - lastUpdate: s.lastUpdate + lastUpdate: s.updatedAt } } else { - listeningStats.items[s.libraryItemId].timeListening += sessionTimeListening + listeningStats.items[libraryItemId].timeListening += sessionTimeListening } listeningStats.totalTime += sessionTimeListening }) + Logger.debug( + `[ApiRouter] Listening stats for user "${userId}" aggregated ${listeningSessions.length} sessions in ${Date.now() - startedAt}ms` + ) return listeningStats } } From 9aa673d0582d981f6dea5c3f06125e9643bf29b1 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 8 Mar 2026 18:28:01 -0400 Subject: [PATCH 25/38] Add minified listening stats mode Before stats on the app were taking about 3422ms and returning roughly 360815 bytes for /api/me/listening-stats. After adding an opt-in minified mode that skips items and recentSessions, the same dataset benchmarks at about 102ms average and 11667 bytes while keeping the default response shape unchanged for existing clients. --- server/controllers/MeController.js | 8 +++-- server/controllers/UserController.js | 8 +++-- server/routers/ApiRouter.js | 49 ++++++++++++++++------------ 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/server/controllers/MeController.js b/server/controllers/MeController.js index 0365a3105..d9a922aa1 100644 --- a/server/controllers/MeController.js +++ b/server/controllers/MeController.js @@ -232,10 +232,14 @@ class MeController { */ async getListeningStats(req, res) { const startedAt = Date.now() + const minified = req.query.minified === '1' || req.query.minified === 'true' Logger.debug(`[MeController] /api/me/listening-stats user="${req.user.id}" start`) - const listeningStats = await this.getUserListeningStatsHelpers(req.user.id) + const listeningStats = await this.getUserListeningStatsHelpers(req.user.id, { + includeItems: !minified, + includeRecentSessions: !minified + }) Logger.debug( - `[MeController] /api/me/listening-stats user="${req.user.id}" totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms` + `[MeController] /api/me/listening-stats user="${req.user.id}" minified=${minified} totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms` ) res.json(listeningStats) } diff --git a/server/controllers/UserController.js b/server/controllers/UserController.js index 89ceb56b3..96dff644d 100644 --- a/server/controllers/UserController.js +++ b/server/controllers/UserController.js @@ -472,12 +472,16 @@ class UserController { */ async getListeningStats(req, res) { const startedAt = Date.now() + const minified = req.query.minified === '1' || req.query.minified === 'true' Logger.debug( `[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" start` ) - var listeningStats = await this.getUserListeningStatsHelpers(req.params.id) + var listeningStats = await this.getUserListeningStatsHelpers(req.params.id, { + includeItems: !minified, + includeRecentSessions: !minified + }) Logger.debug( - `[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms` + `[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" minified=${minified} totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms` ) res.json(listeningStats) } diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index fcb414816..6487a2151 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -555,27 +555,32 @@ class ApiRouter { return userSessions.sort((a, b) => b.updatedAt - a.updatedAt) } - async getUserListeningStatsHelpers(userId) { + async getUserListeningStatsHelpers(userId, options = {}) { const startedAt = Date.now() const today = date.format(new Date(), 'YYYY-MM-DD') + const includeItems = options.includeItems !== false + const includeRecentSessions = options.includeRecentSessions !== false - const [listeningSessions, recentSessions] = await Promise.all([ - Database.getPlaybackSessionsForStats({ userId }), - Database.getPlaybackSessions({ userId }, { - limit: 10, - offset: 0, - order: [['updatedAt', 'DESC']] - }) - ]) + const tasks = [Database.getPlaybackSessionsForStats({ userId })] + if (includeRecentSessions) { + tasks.push( + Database.getPlaybackSessions({ userId }, { + limit: 10, + offset: 0, + order: [['updatedAt', 'DESC']] + }) + ) + } + const [listeningSessions, recentSessions = []] = await Promise.all(tasks) const listeningStats = { totalTime: 0, - items: {}, days: {}, dayOfWeek: {}, - today: 0, - recentSessions + today: 0 } + if (includeItems) listeningStats.items = {} + if (includeRecentSessions) listeningStats.recentSessions = recentSessions listeningSessions.forEach((s) => { const libraryItemId = s.extraData?.libraryItemId || null let sessionTimeListening = s.timeListening @@ -600,21 +605,23 @@ class ApiRouter { return } - if (!listeningStats.items[libraryItemId]) { - listeningStats.items[libraryItemId] = { - id: libraryItemId, - timeListening: sessionTimeListening, - mediaMetadata: s.mediaMetadata, - lastUpdate: s.updatedAt + if (includeItems) { + if (!listeningStats.items[libraryItemId]) { + listeningStats.items[libraryItemId] = { + id: libraryItemId, + timeListening: sessionTimeListening, + mediaMetadata: s.mediaMetadata, + lastUpdate: s.updatedAt + } + } else { + listeningStats.items[libraryItemId].timeListening += sessionTimeListening } - } else { - listeningStats.items[libraryItemId].timeListening += sessionTimeListening } listeningStats.totalTime += sessionTimeListening }) Logger.debug( - `[ApiRouter] Listening stats for user "${userId}" aggregated ${listeningSessions.length} sessions in ${Date.now() - startedAt}ms` + `[ApiRouter] Listening stats for user "${userId}" aggregated ${listeningSessions.length} sessions in ${Date.now() - startedAt}ms includeItems=${includeItems} includeRecentSessions=${includeRecentSessions}` ) return listeningStats } From 7557387189b8aab1abbcc7501c5b037be54fdf22 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 8 Mar 2026 22:30:29 -0400 Subject: [PATCH 26/38] Fix Postgres listening stats regression Before the new minified listening stats path could return 502 because PlaybackSession.afterFind assumed Sequelize model instances even for raw stats rows, and some payloads could serialize invalid totals. After this hotfix the stats helper returns successfully again, with the current dataset benchmarking at about 124ms for full stats and 101ms for minified stats instead of crashing. --- server/models/PlaybackSession.js | 10 ++++++---- server/routers/ApiRouter.js | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/server/models/PlaybackSession.js b/server/models/PlaybackSession.js index 14303a036..26b702d8c 100644 --- a/server/models/PlaybackSession.js +++ b/server/models/PlaybackSession.js @@ -273,18 +273,20 @@ class PlaybackSession extends Model { if (!Array.isArray(findResult)) findResult = [findResult] for (const instance of findResult) { + const values = instance?.dataValues + if (instance.mediaItemType === 'book' && instance.book !== undefined) { instance.mediaItem = instance.book - instance.dataValues.mediaItem = instance.dataValues.book + if (values) values.mediaItem = values.book } else if (instance.mediaItemType === 'podcastEpisode' && instance.podcastEpisode !== undefined) { instance.mediaItem = instance.podcastEpisode - instance.dataValues.mediaItem = instance.dataValues.podcastEpisode + if (values) values.mediaItem = values.podcastEpisode } // To prevent mistakes: delete instance.book - delete instance.dataValues.book + if (values) delete values.book delete instance.podcastEpisode - delete instance.dataValues.podcastEpisode + if (values) delete values.podcastEpisode } }) } diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index 6487a2151..d6117a04e 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -583,10 +583,10 @@ class ApiRouter { if (includeRecentSessions) listeningStats.recentSessions = recentSessions listeningSessions.forEach((s) => { const libraryItemId = s.extraData?.libraryItemId || null - let sessionTimeListening = s.timeListening - if (typeof sessionTimeListening == 'string') { - sessionTimeListening = Number(sessionTimeListening) - } + const numericListening = Number(s.timeListening) + const sessionTimeListening = Number.isFinite(numericListening) + ? numericListening + : 0 if (s.dayOfWeek) { if (!listeningStats.dayOfWeek[s.dayOfWeek]) listeningStats.dayOfWeek[s.dayOfWeek] = 0 From 1914b9d7e73c16d01c8f372d1689c54b41275d30 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 8 Mar 2026 23:08:33 -0400 Subject: [PATCH 27/38] Fix Postgres stats key mapping Before: the optimized Postgres stats query could aggregate 2152 sessions yet still return totalTime=0, items=0, and empty day buckets because the raw rows exposed lowercase keys instead of the camelCase fields the stats helper expected.\n\nAfter: the stats helper accepts both camelCase model fields and lowercase/raw Postgres fields, restoring totals, item aggregation, and day buckets without giving up the faster raw query path. A validation query against the same account now aggregates about 6.7M seconds across 193 items and 630 active days. --- server/routers/ApiRouter.js | 48 ++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index d6117a04e..a674a2948 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -560,6 +560,27 @@ class ApiRouter { const today = date.format(new Date(), 'YYYY-MM-DD') const includeItems = options.includeItems !== false const includeRecentSessions = options.includeRecentSessions !== false + const getSessionField = (session, key) => { + if (!session || typeof session !== 'object') return undefined + if (session[key] !== undefined) return session[key] + + const lowerKey = key.toLowerCase() + if (session[lowerKey] !== undefined) return session[lowerKey] + + return undefined + } + const getSessionObjectField = (session, key) => { + const value = getSessionField(session, key) + if (!value) return null + if (typeof value === 'string') { + try { + return JSON.parse(value) + } catch (error) { + return null + } + } + return value + } const tasks = [Database.getPlaybackSessionsForStats({ userId })] if (includeRecentSessions) { @@ -582,21 +603,26 @@ class ApiRouter { if (includeItems) listeningStats.items = {} if (includeRecentSessions) listeningStats.recentSessions = recentSessions listeningSessions.forEach((s) => { - const libraryItemId = s.extraData?.libraryItemId || null - const numericListening = Number(s.timeListening) + const extraData = getSessionObjectField(s, 'extraData') + const libraryItemId = extraData?.libraryItemId || extraData?.libraryitemid || null + const sessionDate = getSessionField(s, 'date') + const sessionDayOfWeek = getSessionField(s, 'dayOfWeek') + const sessionUpdatedAt = getSessionField(s, 'updatedAt') + const sessionMediaMetadata = getSessionObjectField(s, 'mediaMetadata') + const numericListening = Number(getSessionField(s, 'timeListening')) const sessionTimeListening = Number.isFinite(numericListening) ? numericListening : 0 - if (s.dayOfWeek) { - if (!listeningStats.dayOfWeek[s.dayOfWeek]) listeningStats.dayOfWeek[s.dayOfWeek] = 0 - listeningStats.dayOfWeek[s.dayOfWeek] += sessionTimeListening + if (sessionDayOfWeek) { + if (!listeningStats.dayOfWeek[sessionDayOfWeek]) listeningStats.dayOfWeek[sessionDayOfWeek] = 0 + listeningStats.dayOfWeek[sessionDayOfWeek] += sessionTimeListening } - if (s.date && sessionTimeListening > 0) { - if (!listeningStats.days[s.date]) listeningStats.days[s.date] = 0 - listeningStats.days[s.date] += sessionTimeListening + if (sessionDate && sessionTimeListening > 0) { + if (!listeningStats.days[sessionDate]) listeningStats.days[sessionDate] = 0 + listeningStats.days[sessionDate] += sessionTimeListening - if (s.date === today) { + if (sessionDate === today) { listeningStats.today += sessionTimeListening } } @@ -610,8 +636,8 @@ class ApiRouter { listeningStats.items[libraryItemId] = { id: libraryItemId, timeListening: sessionTimeListening, - mediaMetadata: s.mediaMetadata, - lastUpdate: s.updatedAt + mediaMetadata: sessionMediaMetadata, + lastUpdate: sessionUpdatedAt } } else { listeningStats.items[libraryItemId].timeListening += sessionTimeListening From 4195e8800ec2fdaf004adca4e6da7b0879885378 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 9 Mar 2026 12:58:16 -0400 Subject: [PATCH 28/38] fix case-insensitive local auth user lookup --- server/models/User.js | 12 ++++++---- test/server/models/User.test.js | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/server/models/User.js b/server/models/User.js index 633fc1cef..f71f80f3b 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -348,11 +348,13 @@ class User extends Model { static async getUserByUsername(username) { if (!username) return null - const cachedUser = userCache.getByUsername(username) + const normalizedUsername = username.toLowerCase() + + const cachedUser = userCache.getByUsername(normalizedUsername) if (cachedUser) return cachedUser const user = await this.findOne({ - where: sequelize.where(sequelize.fn('lower', sequelize.col('username')), username.toLowerCase()), + where: sequelize.where(sequelize.fn('LOWER', sequelize.col('username')), normalizedUsername), include: this.sequelize.models.mediaProgress }) @@ -369,11 +371,13 @@ class User extends Model { static async getUserByEmail(email) { if (!email) return null - const cachedUser = userCache.getByEmail(email) + const normalizedEmail = email.toLowerCase() + + const cachedUser = userCache.getByEmail(normalizedEmail) if (cachedUser) return cachedUser const user = await this.findOne({ - where: sequelize.where(sequelize.fn('lower', sequelize.col('email')), email.toLowerCase()), + where: sequelize.where(sequelize.fn('LOWER', sequelize.col('email')), normalizedEmail), include: this.sequelize.models.mediaProgress }) diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js index 5b935fee6..734c31609 100644 --- a/test/server/models/User.test.js +++ b/test/server/models/User.test.js @@ -3,6 +3,48 @@ const sinon = require('sinon') const User = require('../../../server/models/User') describe('User model', () => { + describe('case-insensitive lookup helpers', () => { + afterEach(() => { + sinon.restore() + }) + + it('should query usernames case-insensitively', async () => { + User.sequelize = { + models: { + mediaProgress: {} + } + } + + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + await User.getUserByUsername('Madison') + + expect(findOneStub.calledOnce).to.equal(true) + const options = findOneStub.firstCall.args[0] + expect(options.where.attribute.fn).to.equal('LOWER') + expect(options.where.attribute.args[0].col).to.equal('username') + expect(options.where.logic).to.equal('madison') + }) + + it('should query emails case-insensitively', async () => { + User.sequelize = { + models: { + mediaProgress: {} + } + } + + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + await User.getUserByEmail('Example.User@Example.com') + + expect(findOneStub.calledOnce).to.equal(true) + const options = findOneStub.firstCall.args[0] + expect(options.where.attribute.fn).to.equal('LOWER') + expect(options.where.attribute.args[0].col).to.equal('email') + expect(options.where.logic).to.equal('example.user@example.com') + }) + }) + describe('getUserByIdOrOldId', () => { let originalSequelize From 07d5450dc7fd7ea86e0401e5e0eeb502ae6a68a0 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 2 Aug 2026 17:55:46 -0400 Subject: [PATCH 29/38] add postgres-native backup support Before, scheduled backups opened the Postgres URL through sqlite3, causing a fatal SQLITE_CANTOPEN restart and leaving no usable database backup. After, Postgres uses pg_dump and pg_restore archives while SQLite keeps its existing format, and SQLite open failures are contained. --- Dockerfile | 1 + server/managers/BackupManager.js | 274 ++++++++++++++++++--- server/objects/Backup.js | 6 +- test/server/managers/BackupManager.test.js | 124 ++++++++++ 4 files changed, 367 insertions(+), 38 deletions(-) create mode 100644 test/server/managers/BackupManager.test.js diff --git a/Dockerfile b/Dockerfile index 816bdd3c3..3ddc1a9d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,6 +50,7 @@ ARG NUSQLITE3_PATH RUN apk add --no-cache --update \ tzdata \ ffmpeg \ + postgresql-client \ tini WORKDIR /app diff --git a/server/managers/BackupManager.js b/server/managers/BackupManager.js index a7b531e62..c91def08c 100644 --- a/server/managers/BackupManager.js +++ b/server/managers/BackupManager.js @@ -1,3 +1,4 @@ +const childProcess = require('child_process') const sqlite3 = require('sqlite3') const Path = require('path') const Logger = require('../Logger') @@ -47,6 +48,30 @@ class BackupManager { return global.ServerSettings.maxBackupSize || Infinity } + get databaseBackupConfig() { + if (Database.isPostgresDialect()) { + return { + dialect: 'postgres', + entryName: 'absdatabase.postgres.dump' + } + } + + return { + dialect: 'sqlite', + entryName: 'absdatabase.sqlite' + } + } + + getBackupDialect(backup) { + if (backup.key === 'postgres') return 'postgres' + if (backup.key === 'sqlite' || !backup.key) return 'sqlite' + return null + } + + getBackupEntryName(dialect) { + return dialect === 'postgres' ? 'absdatabase.postgres.dump' : 'absdatabase.sqlite' + } + async init() { try { const backupsDirExists = await fs.pathExists(this.backupPath) @@ -130,13 +155,6 @@ class BackupManager { await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err)) return res.status(400).send('Failed to read backup file - backup might not be a valid .zip file') } - if (!entries['absdatabase.sqlite']) { - Logger.error(`[BackupManager] Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.`) - await zip.close().catch(() => {}) - await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err)) - return res.status(500).send('Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.') - } - const detailsEntry = entries['details'] if (!detailsEntry) { Logger.error('[BackupManager] Invalid backup - missing details entry') @@ -151,10 +169,26 @@ class BackupManager { return res.status(400).send('Invalid backup file - details entry too large') } - const data = await zip.entryData('details') - const details = data.toString('utf8').split('\n') + let backup + try { + const data = await zip.entryData('details') + const details = data.toString('utf8').split('\n') + backup = new Backup({ details, fullPath: tempPath }) + } catch (error) { + Logger.error(`[BackupManager] Invalid backup with no readable details file`, tempPath, error) + await zip.close().catch(() => {}) + await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err)) + return res.status(400).send('Invalid backup file. Missing readable details.') + } - const backup = new Backup({ details, fullPath: tempPath }) + const backupDialect = this.getBackupDialect(backup) + const databaseEntryName = this.getBackupEntryName(backupDialect) + if (!backupDialect || !entries[databaseEntryName]) { + Logger.error(`[BackupManager] Invalid backup with no ${databaseEntryName} file - unsupported database backup.`) + await zip.close().catch(() => {}) + await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err)) + return res.status(500).send(`Invalid backup file. Does not include ${databaseEntryName}.`) + } if (!backup.serverVersion) { Logger.error(`[BackupManager] Invalid backup with no server version - might be a backup created before version 2.0.0`) @@ -204,8 +238,24 @@ class BackupManager { const entries = await zip.entries() + const backupDialect = this.getBackupDialect(backup) + const currentDialect = Database.isPostgresDialect() ? 'postgres' : 'sqlite' + if (!backupDialect) { + await zip.close() + return res.status(500).send('Invalid backup file. Unsupported database backup format.') + } + + if (backupDialect !== currentDialect) { + await zip.close() + return res.status(400).send(`Cannot apply a ${backupDialect} backup while using the ${currentDialect} database.`) + } + + if (backupDialect === 'postgres') { + return this.requestApplyPostgresBackup(apiCacheManager, backup, zip, entries, res) + } + // Ensure backup has an absdatabase.sqlite file - if (!Object.keys(entries).includes('absdatabase.sqlite')) { + if (!Object.keys(entries).includes(this.getBackupEntryName(backupDialect))) { Logger.error(`[BackupManager] Cannot apply old backup ${backup.fullPath}`) await zip.close() return res.status(500).send('Invalid backup file. Does not include absdatabase.sqlite. This might be from an older Audiobookshelf server.') @@ -266,6 +316,70 @@ class BackupManager { SocketAuthority.emitter('backup_applied') } + async requestApplyPostgresBackup(apiCacheManager, backup, zip, entries, res) { + const databaseEntryName = this.getBackupEntryName('postgres') + if (!Object.keys(entries).includes(databaseEntryName)) { + Logger.error(`[BackupManager] Cannot apply Postgres backup ${backup.fullPath}`) + await zip.close() + return res.status(500).send(`Invalid backup file. Does not include ${databaseEntryName}.`) + } + + const tempDumpPath = Path.join(global.ConfigPath, 'absdatabase-postgres-temp.dump') + let reconnected = false + let zipClosed = false + + const closeZip = async () => { + if (zipClosed) return + zipClosed = true + await zip.close() + } + + try { + await fs.remove(tempDumpPath) + await zip.extract(databaseEntryName, tempDumpPath) + + if (!(await fs.pathExists(tempDumpPath))) { + await closeZip() + return res.status(500).send('Failed to extract Postgres database dump from backup') + } + + await Database.disconnect() + await this.restorePostgresDb(tempDumpPath) + + await fs.ensureDir(this.ItemsMetadataPath) + await zip.extract('metadata-items/', this.ItemsMetadataPath) + await fs.ensureDir(this.AuthorsMetadataPath) + await zip.extract('metadata-authors/', this.AuthorsMetadataPath) + await closeZip() + + await Database.reconnect() + reconnected = true + + await apiCacheManager.reset() + await CacheManager.purgeAll() + + res.sendStatus(200) + SocketAuthority.emitter('backup_applied') + } catch (error) { + Logger.error(`[BackupManager] Failed to apply Postgres backup`, error) + try { + await closeZip() + } catch (closeError) { + Logger.error(`[BackupManager] Failed to close Postgres backup archive`, closeError) + } + if (!reconnected) { + try { + await Database.reconnect() + } catch (reconnectError) { + Logger.error(`[BackupManager] Failed to reconnect after Postgres backup apply`, reconnectError) + } + } + return res.status(500).send(`Failed to apply Postgres backup: ${error?.message || 'Unknown Error'}`) + } finally { + await fs.remove(tempDumpPath) + } + } + async loadBackups() { try { const filesInDir = await fs.readdir(this.backupPath) @@ -303,6 +417,14 @@ class BackupManager { const details = data.toString('utf8').split('\n') const backup = new Backup({ details, fullPath: fullFilePath }) + const backupDialect = this.getBackupDialect(backup) + const databaseEntryName = this.getBackupEntryName(backupDialect) + + if (!backupDialect || !Object.keys(await zip.entries()).includes(databaseEntryName)) { + Logger.error(`[BackupManager] Unsupported database backup format found "${backup.filename}"`) + await zip.close() + continue + } if (!backup.serverVersion) { // Backups before v2 @@ -333,33 +455,34 @@ class BackupManager { async runBackup() { // Check if Metadata Path is inside Config Path (otherwise there will be an infinite loop as the archiver tries to zip itself) Logger.info(`[BackupManager] Running Backup`) + const databaseBackupConfig = this.databaseBackupConfig const newBackup = new Backup() - newBackup.setData(this.backupPath) + newBackup.setData(this.backupPath, databaseBackupConfig.dialect) await fs.ensureDir(this.AuthorsMetadataPath) - // Create backup sqlite file - const sqliteBackupPath = await this.backupSqliteDb(newBackup).catch((error) => { - Logger.error(`[BackupManager] Failed to backup sqlite db`, error) + // Create a database dump + const databaseBackupPath = await this.backupDatabase(newBackup).catch((error) => { + Logger.error(`[BackupManager] Failed to backup ${databaseBackupConfig.dialect} database`, error) const errorMsg = error?.message || error || 'Unknown Error' NotificationManager.onBackupFailed(errorMsg) return false }) - if (!sqliteBackupPath) { + if (!databaseBackupPath) { return false } - // Zip sqlite file, /metadata/items, and /metadata/authors folders - const zipResult = await this.zipBackup(sqliteBackupPath, newBackup).catch((error) => { + // Zip database dump, /metadata/items, and /metadata/authors folders + const zipResult = await this.zipBackup(databaseBackupPath, newBackup, databaseBackupConfig.entryName).catch((error) => { Logger.error(`[BackupManager] Backup Failed ${error}`) const errorMsg = error?.message || error || 'Unknown Error' NotificationManager.onBackupFailed(errorMsg) return false }) - // Remove sqlite backup - await fs.remove(sqliteBackupPath) + // Remove temporary database dump + await fs.remove(databaseBackupPath) if (!zipResult) return false @@ -390,6 +513,10 @@ class BackupManager { return true } + backupDatabase(backup) { + return this.databaseBackupConfig.dialect === 'postgres' ? this.backupPostgresDb(backup) : this.backupSqliteDb(backup) + } + async removeBackup(backup) { try { Logger.debug(`[BackupManager] Removing Backup "${backup.fullPath}"`) @@ -406,29 +533,106 @@ class BackupManager { * @param {Backup} backup */ backupSqliteDb(backup) { - const db = new sqlite3.Database(Database.dbPath) const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.sqlite`) return new Promise(async (resolve, reject) => { - const backup = db.backup(dbFilePath) - backup.step(-1) - backup.finish() + let db + let sqliteBackup + let settled = false - // Max time ~2 mins - for (let i = 0; i < 240; i++) { - if (backup.completed) { - return resolve(dbFilePath) - } else if (backup.failed) { - return reject(backup.message || 'Unknown failure reason') + const finish = (error, result) => { + if (settled) return + settled = true + if (db) { + db.close(() => { + if (error) reject(error) + else resolve(result) + }) + } else if (error) { + reject(error) + } else { + resolve(result) } - await new Promise((r) => setTimeout(r, 500)) } - Logger.error(`[BackupManager] Backup sqlite timed out`) - reject('Backup timed out') + const pollBackup = async () => { + // Max time ~2 mins + for (let i = 0; i < 240; i++) { + if (sqliteBackup.completed) { + return finish(null, dbFilePath) + } else if (sqliteBackup.failed) { + return finish(sqliteBackup.message || 'Unknown failure reason') + } + await new Promise((r) => setTimeout(r, 500)) + } + + Logger.error(`[BackupManager] Backup sqlite timed out`) + finish('Backup timed out') + } + + const startBackup = () => { + try { + sqliteBackup = db.backup(dbFilePath) + sqliteBackup.step(-1) + sqliteBackup.finish() + pollBackup().catch(finish) + } catch (error) { + finish(error) + } + } + + db = new sqlite3.Database(Database.dbPath, (error) => { + if (error) return finish(error) + startBackup() + }) + db.on('error', finish) }) } - zipBackup(sqliteBackupPath, backup) { + backupPostgresDb(backup) { + const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.postgres.dump`) + return this.runPostgresCommand('pg_dump', [ + '--format=custom', + '--no-owner', + '--no-acl', + '--file', + dbFilePath, + '--dbname', + Database.dbPath + ]) + .then(() => dbFilePath) + .catch(async (error) => { + await fs.remove(dbFilePath) + throw error + }) + } + + restorePostgresDb(dbFilePath) { + return this.runPostgresCommand('pg_restore', [ + '--clean', + '--if-exists', + '--exit-on-error', + '--single-transaction', + '--no-owner', + '--no-acl', + '--dbname', + Database.dbPath, + dbFilePath + ]) + } + + runPostgresCommand(command, args) { + return new Promise((resolve, reject) => { + childProcess.execFile(command, args, { maxBuffer: 10 * 1024 * 1024 }, (error, stdout, stderr) => { + if (error) { + error.stderr = stderr + return reject(error) + } + resolve({ stdout, stderr }) + }) + }) + } + + zipBackup(databaseBackupPath, backup, databaseEntryName = 'absdatabase.sqlite') { return new Promise((resolve, reject) => { // create a file to stream archive data to const output = fs.createWriteStream(backup.fullPath) @@ -492,7 +696,7 @@ class BackupManager { // pipe archive data to the file archive.pipe(output) - archive.file(sqliteBackupPath, { name: 'absdatabase.sqlite' }) + archive.file(databaseBackupPath, { name: databaseEntryName }) archive.directory(this.ItemsMetadataPath, 'metadata-items') archive.directory(this.AuthorsMetadataPath, 'metadata-authors') diff --git a/server/objects/Backup.js b/server/objects/Backup.js index e3b9f4b4a..e12c294c5 100644 --- a/server/objects/Backup.js +++ b/server/objects/Backup.js @@ -62,9 +62,9 @@ class Backup { } } - setData(backupDirPath) { + setData(backupDirPath, dialect = 'sqlite') { this.id = date.format(new Date(), 'YYYY-MM-DD[T]HHmm') - this.key = 'sqlite' + this.key = dialect this.datePretty = date.format(new Date(), 'ddd, MMM D YYYY HH:mm') this.backupDirPath = backupDirPath @@ -78,4 +78,4 @@ class Backup { this.createdAt = Date.now() } } -module.exports = Backup \ No newline at end of file +module.exports = Backup diff --git a/test/server/managers/BackupManager.test.js b/test/server/managers/BackupManager.test.js new file mode 100644 index 000000000..c17a529a1 --- /dev/null +++ b/test/server/managers/BackupManager.test.js @@ -0,0 +1,124 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const os = require('os') +const Path = require('path') +const EventEmitter = require('events') +const childProcess = require('child_process') +const sqlite3 = require('sqlite3') + +const BackupManager = require('../../../server/managers/BackupManager') +const Backup = require('../../../server/objects/Backup') +const Database = require('../../../server/Database') + +describe('BackupManager', () => { + let originalDialect + let originalDbPath + let originalConfigPath + let originalMetadataPath + + beforeEach(() => { + originalDialect = Database.dialect + originalDbPath = Database.dbPath + originalConfigPath = global.ConfigPath + originalMetadataPath = global.MetadataPath + global.MetadataPath = os.tmpdir() + }) + + afterEach(() => { + Database.dialect = originalDialect + Database.dbPath = originalDbPath + global.ConfigPath = originalConfigPath + global.MetadataPath = originalMetadataPath + sinon.restore() + }) + + it('should select Postgres custom-format backups for the Postgres dialect', () => { + Database.dialect = 'postgres' + const manager = new BackupManager() + + expect(manager.databaseBackupConfig).to.deep.equal({ + dialect: 'postgres', + entryName: 'absdatabase.postgres.dump' + }) + }) + + it('should create Postgres dumps with pg_dump and preserve the connection URL', async () => { + Database.dialect = 'postgres' + Database.dbPath = 'postgresql://localhost:5432/audiobookshelf' + global.ConfigPath = os.tmpdir() + + const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => { + callback(null, '', '') + }) + const manager = new BackupManager() + const backup = new Backup() + backup.id = '2026-08-02T0130' + + const dumpPath = await manager.backupPostgresDb(backup) + + expect(dumpPath).to.equal(Path.join(os.tmpdir(), 'absdatabase.2026-08-02T0130.postgres.dump')) + expect(execFileStub.calledOnce).to.equal(true) + expect(execFileStub.firstCall.args[0]).to.equal('pg_dump') + expect(execFileStub.firstCall.args[1]).to.deep.equal([ + '--format=custom', + '--no-owner', + '--no-acl', + '--file', + dumpPath, + '--dbname', + Database.dbPath + ]) + }) + + it('should restore Postgres dumps in one transaction and clean existing objects', async () => { + Database.dialect = 'postgres' + Database.dbPath = 'postgresql://localhost:5432/audiobookshelf' + + const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => { + callback(null, '', '') + }) + const manager = new BackupManager() + + await manager.restorePostgresDb('/config/absdatabase-postgres-temp.dump') + + expect(execFileStub.firstCall.args[0]).to.equal('pg_restore') + expect(execFileStub.firstCall.args[1]).to.deep.equal([ + '--clean', + '--if-exists', + '--exit-on-error', + '--single-transaction', + '--no-owner', + '--no-acl', + '--dbname', + Database.dbPath, + '/config/absdatabase-postgres-temp.dump' + ]) + }) + + it('should reject SQLite backup open errors without an uncaught sqlite event', async () => { + Database.dialect = 'sqlite' + Database.dbPath = '/config/absdatabase.sqlite' + global.ConfigPath = os.tmpdir() + + sinon.stub(sqlite3, 'Database').callsFake(function (_dbPath, callback) { + const db = new EventEmitter() + db.close = (closeCallback) => closeCallback() + process.nextTick(() => callback(Object.assign(new Error('unable to open database file'), { code: 'SQLITE_CANTOPEN' }))) + return db + }) + + const manager = new BackupManager() + const backup = new Backup() + backup.id = '2026-08-02T0130' + + let error + try { + await manager.backupSqliteDb(backup) + } catch (caughtError) { + error = caughtError + } + + expect(error).to.be.an('error') + expect(error.code).to.equal('SQLITE_CANTOPEN') + }) +}) From 8e6a1d57d7e23c087d84301932ef290a4843ae20 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 2 Aug 2026 17:56:08 -0400 Subject: [PATCH 30/38] harden postgres migration integer safety Before, integer preflight accepted values through JavaScript Number and the migration could truncate a target without an explicit confirmation. After, integer bounds and bigint strings are checked without precision loss, and destructive writes require ALLOW_DESTRUCTIVE_TARGET=true. --- server/scripts/migrateSqliteToPostgres.js | 51 ++++++++++++++----- .../scripts/migrateSqliteToPostgres.test.js | 5 +- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js index 43dc439da..1b6adcd7d 100644 --- a/server/scripts/migrateSqliteToPostgres.js +++ b/server/scripts/migrateSqliteToPostgres.js @@ -8,6 +8,13 @@ const DATABASE_URL = process.env.DATABASE_URL const PG_SCHEMA = process.env.PG_SCHEMA || 'public' const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500) const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true' +const ALLOW_DESTRUCTIVE_TARGET = String(process.env.ALLOW_DESTRUCTIVE_TARGET || 'false').toLowerCase() === 'true' + +const integerBounds = { + smallint: { min: -32768n, max: 32767n }, + integer: { min: -2147483648n, max: 2147483647n }, + bigint: { min: -9223372036854775808n, max: 9223372036854775807n } +} const preferredOrder = [ 'migrationsMeta', @@ -117,22 +124,34 @@ async function findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByT return issues } -function isIntegerCompatible(value) { - if (value === null || value === undefined) return true - +function parseIntegerValue(value) { if (typeof value === 'number') { - return Number.isFinite(value) && Number.isInteger(value) + if (!Number.isSafeInteger(value)) return null + return BigInt(value) } if (typeof value === 'string') { const trimmed = value.trim() - if (!trimmed) return false - if (!/^-?\d+$/.test(trimmed)) return false - const parsed = Number(trimmed) - return Number.isFinite(parsed) + if (!trimmed || !/^-?\d+$/.test(trimmed)) return null + try { + return BigInt(trimmed) + } catch (error) { + return null + } } - return false + return null +} + +function isIntegerCompatible(value, dataType = 'bigint') { + if (value === null || value === undefined) return true + + const parsed = parseIntegerValue(value) + if (parsed === null) return false + + const bounds = integerBounds[dataType] + if (!bounds) return false + return parsed >= bounds.min && parsed <= bounds.max } async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) { @@ -161,7 +180,7 @@ async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable let badCount = 0 let sampleValue = null for (const row of rows) { - if (!isIntegerCompatible(row.value)) { + if (!isIntegerCompatible(row.value, dataType)) { badCount += 1 if (sampleValue === null) sampleValue = row.value } @@ -231,10 +250,12 @@ function convertValue(value, pgColumn) { } if ((dataType === 'smallint' || dataType === 'integer' || dataType === 'bigint') && value !== null && value !== undefined) { - if (typeof value === 'number') return value - if (typeof value === 'string' && /^-?\d+$/.test(value.trim())) { - return Number(value) + if (!isIntegerCompatible(value, dataType)) return value + if (dataType === 'bigint' && typeof value === 'string') { + return value.trim() } + if (typeof value === 'number') return value + return Number(value) } return value @@ -353,6 +374,10 @@ async function main() { } if (!DRY_RUN) { + if (!ALLOW_DESTRUCTIVE_TARGET) { + throw new Error('Migration writes are destructive. Set ALLOW_DESTRUCTIVE_TARGET=true after confirming the target database can be truncated.') + } + await pg.query('BEGIN') await pg.query('SET session_replication_role = replica') diff --git a/test/server/scripts/migrateSqliteToPostgres.test.js b/test/server/scripts/migrateSqliteToPostgres.test.js index d0bb3d592..f3049ab62 100644 --- a/test/server/scripts/migrateSqliteToPostgres.test.js +++ b/test/server/scripts/migrateSqliteToPostgres.test.js @@ -128,11 +128,14 @@ describe('migrateSqliteToPostgres script helpers', () => { expect(isIntegerCompatible('10')).to.equal(true) expect(isIntegerCompatible(10.5)).to.equal(false) expect(isIntegerCompatible('10.5')).to.equal(false) + expect(isIntegerCompatible('32768', 'smallint')).to.equal(false) + expect(isIntegerCompatible('9007199254740993', 'bigint')).to.equal(true) }) it('should coerce integer-like strings for postgres integer columns', () => { expect(convertValue('42', { data_type: 'integer' })).to.equal(42) - expect(convertValue('-7', { data_type: 'bigint' })).to.equal(-7) + expect(convertValue('-7', { data_type: 'bigint' })).to.equal('-7') + expect(convertValue('9007199254740993', { data_type: 'bigint' })).to.equal('9007199254740993') expect(convertValue('4.2', { data_type: 'integer' })).to.equal('4.2') }) From c8f753c58c12252a702cc4796ecb7d4b1d42449d Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 2 Aug 2026 18:14:58 -0400 Subject: [PATCH 31/38] keep postgres backup credentials out of command argv Pass connection parts as individual pg_dump/pg_restore args and the password via PGPASSWORD env so credentials never appear in argv, execFile error messages, notifications, or the host process list. Redact the password from any error output as a fallback, and reject non-URI DATABASE_URL values with a clear error. --- server/managers/BackupManager.js | 53 +++++++++++++-- test/server/managers/BackupManager.test.js | 76 ++++++++++++++++++++-- 2 files changed, 116 insertions(+), 13 deletions(-) diff --git a/server/managers/BackupManager.js b/server/managers/BackupManager.js index c91def08c..e27c2f111 100644 --- a/server/managers/BackupManager.js +++ b/server/managers/BackupManager.js @@ -588,6 +588,31 @@ class BackupManager { }) } + /** + * Build pg_dump/pg_restore connection arguments from DATABASE_URL without + * exposing credentials in argv. execFile error messages and the host process + * list include argv, so the password is passed via PGPASSWORD env instead. + */ + getPostgresConnection() { + let dbUrl + try { + dbUrl = new URL(Database.dbPath) + } catch (error) { + throw new Error('DATABASE_URL must be a valid postgres connection URI to run backups') + } + + const args = ['--host', dbUrl.hostname, '--dbname', decodeURIComponent(dbUrl.pathname.replace(/^\//, ''))] + if (dbUrl.port) args.push('--port', dbUrl.port) + if (dbUrl.username) args.push('--username', decodeURIComponent(dbUrl.username)) + + // Redact both the percent-encoded and decoded password from any error output + const decodedPassword = dbUrl.password ? decodeURIComponent(dbUrl.password) : null + const secrets = dbUrl.password ? [dbUrl.password, decodedPassword] : [] + const env = decodedPassword ? { ...process.env, PGPASSWORD: decodedPassword } : process.env + + return { args, env, secrets } + } + backupPostgresDb(backup) { const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.postgres.dump`) return this.runPostgresCommand('pg_dump', [ @@ -595,9 +620,7 @@ class BackupManager { '--no-owner', '--no-acl', '--file', - dbFilePath, - '--dbname', - Database.dbPath + dbFilePath ]) .then(() => dbFilePath) .catch(async (error) => { @@ -614,17 +637,33 @@ class BackupManager { '--single-transaction', '--no-owner', '--no-acl', - '--dbname', - Database.dbPath, dbFilePath ]) } runPostgresCommand(command, args) { return new Promise((resolve, reject) => { - childProcess.execFile(command, args, { maxBuffer: 10 * 1024 * 1024 }, (error, stdout, stderr) => { + let connection + try { + connection = this.getPostgresConnection() + } catch (error) { + return reject(error) + } + + const redact = (text) => { + if (typeof text !== 'string') return text + return connection.secrets.reduce((redacted, secret) => redacted.split(secret).join('***'), text) + } + + const options = { + maxBuffer: 10 * 1024 * 1024, + env: connection.env + } + childProcess.execFile(command, [...args, ...connection.args], options, (error, stdout, stderr) => { if (error) { - error.stderr = stderr + error.message = redact(error.message) + if (error.cmd) error.cmd = redact(error.cmd) + error.stderr = redact(stderr) return reject(error) } resolve({ stdout, stderr }) diff --git a/test/server/managers/BackupManager.test.js b/test/server/managers/BackupManager.test.js index c17a529a1..b5bbcc311 100644 --- a/test/server/managers/BackupManager.test.js +++ b/test/server/managers/BackupManager.test.js @@ -42,9 +42,9 @@ describe('BackupManager', () => { }) }) - it('should create Postgres dumps with pg_dump and preserve the connection URL', async () => { + it('should create Postgres dumps with pg_dump without exposing credentials in argv', async () => { Database.dialect = 'postgres' - Database.dbPath = 'postgresql://localhost:5432/audiobookshelf' + Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf' global.ConfigPath = os.tmpdir() const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => { @@ -65,14 +65,22 @@ describe('BackupManager', () => { '--no-acl', '--file', dumpPath, + '--host', + 'localhost', '--dbname', - Database.dbPath + 'audiobookshelf', + '--port', + '5432', + '--username', + 'absuser' ]) + expect(execFileStub.firstCall.args[1].join(' ')).to.not.include('secretpass') + expect(execFileStub.firstCall.args[2].env.PGPASSWORD).to.equal('secretpass') }) it('should restore Postgres dumps in one transaction and clean existing objects', async () => { Database.dialect = 'postgres' - Database.dbPath = 'postgresql://localhost:5432/audiobookshelf' + Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf' const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => { callback(null, '', '') @@ -89,10 +97,66 @@ describe('BackupManager', () => { '--single-transaction', '--no-owner', '--no-acl', + '/config/absdatabase-postgres-temp.dump', + '--host', + 'localhost', '--dbname', - Database.dbPath, - '/config/absdatabase-postgres-temp.dump' + 'audiobookshelf', + '--port', + '5432', + '--username', + 'absuser' ]) + expect(execFileStub.firstCall.args[1].join(' ')).to.not.include('secretpass') + expect(execFileStub.firstCall.args[2].env.PGPASSWORD).to.equal('secretpass') + }) + + it('should redact database credentials from failed pg command errors', async () => { + Database.dialect = 'postgres' + Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf' + global.ConfigPath = os.tmpdir() + + sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => { + const error = new Error('Command failed: pg_dump --dbname postgresql://absuser:secretpass@localhost/audiobookshelf\npg_dump: error: password authentication failed') + error.cmd = 'pg_dump --dbname postgresql://absuser:secretpass@localhost/audiobookshelf' + callback(error, '', 'connection using password secretpass failed') + }) + const manager = new BackupManager() + const backup = new Backup() + backup.id = '2026-08-02T0130' + + let error + try { + await manager.backupPostgresDb(backup) + } catch (caughtError) { + error = caughtError + } + + expect(error).to.be.an('error') + expect(error.message).to.not.include('secretpass') + expect(error.cmd).to.not.include('secretpass') + expect(error.stderr).to.not.include('secretpass') + expect(error.message).to.include('***') + }) + + it('should reject pg commands when DATABASE_URL is not a valid URI', async () => { + Database.dialect = 'postgres' + Database.dbPath = 'not a connection uri' + global.ConfigPath = os.tmpdir() + + const manager = new BackupManager() + const backup = new Backup() + backup.id = '2026-08-02T0130' + + let error + try { + await manager.backupPostgresDb(backup) + } catch (caughtError) { + error = caughtError + } + + expect(error).to.be.an('error') + expect(error.message).to.include('valid postgres connection URI') }) it('should reject SQLite backup open errors without an uncaught sqlite event', async () => { From dedbe57b0ca2158cd724b3afe951ff1bbdd07864 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 2 Aug 2026 18:15:34 -0400 Subject: [PATCH 32/38] add timeout to postgres backup commands Kill pg_dump/pg_restore after 30 minutes so a lock wait cannot hang a backup or leave the server disconnected from its database during a restore. A killed restore rolls back via --single-transaction. --- server/managers/BackupManager.js | 2 ++ test/server/managers/BackupManager.test.js | 1 + 2 files changed, 3 insertions(+) diff --git a/server/managers/BackupManager.js b/server/managers/BackupManager.js index e27c2f111..aef76bddc 100644 --- a/server/managers/BackupManager.js +++ b/server/managers/BackupManager.js @@ -657,6 +657,8 @@ class BackupManager { const options = { maxBuffer: 10 * 1024 * 1024, + // Kill after 30 mins (e.g. lock waits) - a killed restore rolls back via --single-transaction + timeout: 30 * 60 * 1000, env: connection.env } childProcess.execFile(command, [...args, ...connection.args], options, (error, stdout, stderr) => { diff --git a/test/server/managers/BackupManager.test.js b/test/server/managers/BackupManager.test.js index b5bbcc311..5c5797814 100644 --- a/test/server/managers/BackupManager.test.js +++ b/test/server/managers/BackupManager.test.js @@ -76,6 +76,7 @@ describe('BackupManager', () => { ]) expect(execFileStub.firstCall.args[1].join(' ')).to.not.include('secretpass') expect(execFileStub.firstCall.args[2].env.PGPASSWORD).to.equal('secretpass') + expect(execFileStub.firstCall.args[2].timeout).to.be.a('number') }) it('should restore Postgres dumps in one transaction and clean existing objects', async () => { From d283d8c332b5147d94319bb4e1b6e2223170c05f Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 2 Aug 2026 18:16:03 -0400 Subject: [PATCH 33/38] make backup listing resilient to corrupt archives A corrupt or unreadable zip no longer aborts the whole loadBackups loop or leaks the zip handle - the entry check now sits inside the per-file try so one bad archive is skipped like any other invalid backup. --- server/managers/BackupManager.js | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/server/managers/BackupManager.js b/server/managers/BackupManager.js index aef76bddc..277982193 100644 --- a/server/managers/BackupManager.js +++ b/server/managers/BackupManager.js @@ -390,7 +390,7 @@ class BackupManager { const fullFilePath = Path.join(this.backupPath, filename) let zip = null - let data = null + let backup = null try { zip = new StreamZip.async({ file: fullFilePath }) const entries = await zip.entries() @@ -407,25 +407,24 @@ class BackupManager { continue } - data = await zip.entryData('details') + const data = await zip.entryData('details') + const details = data.toString('utf8').split('\n') + + backup = new Backup({ details, fullPath: fullFilePath }) + const backupDialect = this.getBackupDialect(backup) + const databaseEntryName = this.getBackupEntryName(backupDialect) + + if (!backupDialect || !entries[databaseEntryName]) { + Logger.error(`[BackupManager] Unsupported database backup format found "${backup.filename}"`) + await zip.close().catch(() => {}) + continue + } } catch (error) { Logger.error(`[BackupManager] Failed to unzip backup "${fullFilePath}"`, error) if (zip) await zip.close().catch(() => {}) continue } - const details = data.toString('utf8').split('\n') - - const backup = new Backup({ details, fullPath: fullFilePath }) - const backupDialect = this.getBackupDialect(backup) - const databaseEntryName = this.getBackupEntryName(backupDialect) - - if (!backupDialect || !Object.keys(await zip.entries()).includes(databaseEntryName)) { - Logger.error(`[BackupManager] Unsupported database backup format found "${backup.filename}"`) - await zip.close() - continue - } - if (!backup.serverVersion) { // Backups before v2 Logger.error(`[BackupManager] Old unsupported backup was found "${backup.filename}"`) From 9abf86a5b96b218b0d2e90f54bc81cb0e5db4d1c Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 2 Aug 2026 18:39:27 -0400 Subject: [PATCH 34/38] add postgres denormalization triggers for library items The sqlite triggers that keep libraryItems.title[IgnorePrefix] and authorNamesFirstLast/LastFirst fresh were skipped on postgres, so title edits, author renames, and bookAuthor changes left stale sort/search columns. Mirror them with native pg trigger functions (string_agg in place of GROUP_CONCAT) using folded lowercase identifiers, and keep the same trigger names and existence checks. Validated against PostgreSQL 14: title propagation, author insert/delete/rename, createdat ordering, and null-on-empty all match sqlite semantics. --- server/Database.js | 118 +++++++++++++++++++++++++++++++++++ test/server/Database.test.js | 59 ++++++++++++++++++ 2 files changed, 177 insertions(+) diff --git a/server/Database.js b/server/Database.js index 247c7e974..df0d6939b 100644 --- a/server/Database.js +++ b/server/Database.js @@ -928,6 +928,9 @@ WHERE EXISTS ( * It adds triggers to update libraryItems.title[IgnorePrefix] when (books|podcasts).title[IgnorePrefix] is updated */ async addTriggers() { + if (this.isPostgresDialect()) { + return this.addPostgresTriggers() + } if (!this.isSqliteDialect()) { Logger.info(`[Database] Skipping sqlite-only triggers for dialect ${this.dialect}`) return @@ -1030,6 +1033,121 @@ WHERE EXISTS ( await addAuthorsUpdateTriggerIfNotExists() } + /** + * Postgres equivalent of the sqlite libraryItems denormalization triggers above. + * Identifiers are unquoted (and therefore folded to lowercase by postgres) to match + * the quoteIdentifiers: false strategy used for the postgres dialect. + */ + async addPostgresTriggers() { + Logger.info('[Database] Adding postgres denormalization triggers') + + await this.addPostgresTitleTriggerIfNotExists('books', 'title') + await this.addPostgresTitleTriggerIfNotExists('books', 'titleIgnorePrefix') + await this.addPostgresTitleTriggerIfNotExists('podcasts', 'title') + await this.addPostgresTitleTriggerIfNotExists('podcasts', 'titleIgnorePrefix') + await this.addPostgresAuthorNamesTriggersIfNotExist() + } + + async postgresTriggerExists(triggerName) { + const [[{ count }]] = await this.sequelize.query(`SELECT COUNT(*) as count FROM pg_trigger WHERE NOT tgisinternal AND tgname = '${triggerName}'`) + return Number(count) > 0 + } + + async addPostgresTitleTriggerIfNotExists(sourceTable, sourceColumn) { + const foldedColumn = sourceColumn.toLowerCase() + const action = `update_libraryItems_${sourceColumn}` + const fromSource = sourceTable === 'books' ? '' : `_from_${sourceTable}_${sourceColumn}` + const triggerName = this.convertToSnakeCase(`${action}${fromSource}`) + const functionName = `${triggerName}_fn` + + if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists + + Logger.info(`[Database] Adding trigger ${triggerName}`) + + await this.sequelize.query(` + CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$ + BEGIN + UPDATE libraryitems + SET ${foldedColumn} = NEW.${foldedColumn} + WHERE mediaid = NEW.id; + RETURN NEW; + END; + $func$ LANGUAGE plpgsql + `) + await this.sequelize.query(` + CREATE TRIGGER ${triggerName} + AFTER UPDATE OF ${foldedColumn} ON ${sourceTable} + FOR EACH ROW + EXECUTE FUNCTION ${functionName}() + `) + } + + async addPostgresAuthorNamesTriggersIfNotExist() { + // string_agg is the postgres equivalent of sqlite GROUP_CONCAT; both return NULL for an empty set + const authorNamesSubQuery = (bookIdExpression) => ` + SELECT string_agg(authors.name, ', ' ORDER BY bookauthors.createdat ASC), string_agg(authors.lastfirst, ', ' ORDER BY bookauthors.createdat ASC) + FROM authors JOIN bookauthors ON authors.id = bookauthors.authorid + WHERE bookauthors.bookid = ${bookIdExpression} + ` + + const addBookAuthorsTriggerIfNotExists = async (action) => { + const modifiedRecord = action === 'delete' ? 'OLD' : 'NEW' + const triggerName = this.convertToSnakeCase(`update_libraryItems_authorNames_on_bookAuthors_${action}`) + const functionName = `${triggerName}_fn` + + if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists + + Logger.info(`[Database] Adding trigger ${triggerName}`) + + await this.sequelize.query(` + CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$ + BEGIN + UPDATE libraryitems + SET (authornamesfirstlast, authornameslastfirst) = (${authorNamesSubQuery(`${modifiedRecord}.bookid`)}) + WHERE mediaid = ${modifiedRecord}.bookid; + RETURN ${modifiedRecord}; + END; + $func$ LANGUAGE plpgsql + `) + await this.sequelize.query(` + CREATE TRIGGER ${triggerName} + AFTER ${action.toUpperCase()} ON bookauthors + FOR EACH ROW + EXECUTE FUNCTION ${functionName}() + `) + } + + const addAuthorsUpdateTriggerIfNotExists = async () => { + const triggerName = this.convertToSnakeCase('update_libraryItems_authorNames_on_authors_update') + const functionName = `${triggerName}_fn` + + if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists + + Logger.info(`[Database] Adding trigger ${triggerName}`) + + await this.sequelize.query(` + CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$ + BEGIN + UPDATE libraryitems + SET (authornamesfirstlast, authornameslastfirst) = (${authorNamesSubQuery('libraryitems.mediaid')}) + WHERE mediaid IN (SELECT bookid FROM bookauthors WHERE authorid = NEW.id); + RETURN NEW; + END; + $func$ LANGUAGE plpgsql + `) + await this.sequelize.query(` + CREATE TRIGGER ${triggerName} + AFTER UPDATE OF name ON authors + FOR EACH ROW + EXECUTE FUNCTION ${functionName}() + `) + } + + await addBookAuthorsTriggerIfNotExists('insert') + await addBookAuthorsTriggerIfNotExists('delete') + await addAuthorsUpdateTriggerIfNotExists() + } + convertToSnakeCase(str) { return str.replace(/([A-Z])/g, '_$1').toLowerCase() } diff --git a/test/server/Database.test.js b/test/server/Database.test.js index a70e1d204..047d42a31 100644 --- a/test/server/Database.test.js +++ b/test/server/Database.test.js @@ -98,4 +98,63 @@ describe('Database', () => { expect(hasTables).to.equal(false) }) }) + + describe('addPostgresTriggers', () => { + function captureQueries(existingTriggers = []) { + const queries = [] + Database.sequelize = { + query: async (sql) => { + queries.push(sql) + const count = existingTriggers.filter((name) => sql.includes(`tgname = '${name}'`)).length + return [[{ count }]] + } + } + return queries + } + + it('should create title and author names triggers with folded lowercase identifiers', async () => { + const queries = captureQueries() + + await Database.addPostgresTriggers() + + const functions = queries.filter((sql) => sql.includes('CREATE OR REPLACE FUNCTION')) + const triggers = queries.filter((sql) => sql.includes('CREATE TRIGGER')) + expect(functions.length).to.equal(7) + expect(triggers.length).to.equal(7) + + const allDdl = [...functions, ...triggers].join('\n') + // No camelCase identifiers may leak into postgres DDL - unquoted identifiers fold to lowercase + expect(allDdl).to.not.match(/libraryItems|bookAuthors|mediaId|titleIgnorePrefix|authorNames|bookId|authorId|lastFirst|createdAt/) + + const authorTrigger = triggers.find((sql) => sql.includes('update_library_items_author_names_on_authors_update')) + expect(authorTrigger).to.include('AFTER UPDATE OF name ON authors') + + const insertFn = functions.find((sql) => sql.includes('update_library_items_author_names_on_book_authors_insert_fn')) + expect(insertFn).to.include("string_agg(authors.name, ', ' ORDER BY bookauthors.createdat ASC)") + expect(insertFn).to.include('WHERE mediaid = NEW.bookid') + + const deleteFn = functions.find((sql) => sql.includes('update_library_items_author_names_on_book_authors_delete_fn')) + expect(deleteFn).to.include('WHERE mediaid = OLD.bookid') + }) + + it('should skip triggers that already exist', async () => { + const queries = captureQueries(['update_library_items_title']) + + await Database.addPostgresTriggers() + + const titleFn = queries.find((sql) => sql.includes('update_library_items_title_fn')) + expect(titleFn).to.equal(undefined) + const otherFns = queries.filter((sql) => sql.includes('CREATE OR REPLACE FUNCTION')) + expect(otherFns.length).to.equal(6) + }) + + it('should dispatch to postgres triggers from addTriggers', async () => { + Database.dialect = 'postgres' + const queries = captureQueries() + + await Database.addTriggers() + + expect(queries.some((sql) => sql.includes('CREATE TRIGGER'))).to.equal(true) + }) + }) }) From a9c89e48288b3a1a863c0763f5fdfcbce84d0711 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 2 Aug 2026 18:39:42 -0400 Subject: [PATCH 35/38] make v2.35.0 migration column checks case-insensitive describeTable on a migrated postgres database returns lowercase column names, so the camelCase existence checks never matched and a replay attempted addColumn twice, failing the migration. Match column names case-insensitively like the other postgres-hardened migrations. --- .../v2.35.0-add-last-refresh-token.js | 12 ++- .../v2.35.0-add-last-refresh-token.test.js | 98 +++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 test/server/migrations/v2.35.0-add-last-refresh-token.test.js diff --git a/server/migrations/v2.35.0-add-last-refresh-token.js b/server/migrations/v2.35.0-add-last-refresh-token.js index 0ad190e9a..a689c2914 100644 --- a/server/migrations/v2.35.0-add-last-refresh-token.js +++ b/server/migrations/v2.35.0-add-last-refresh-token.js @@ -22,8 +22,10 @@ async function up({ context: { queryInterface, logger } }) { if (await queryInterface.tableExists('sessions')) { const tableDescription = await queryInterface.describeTable('sessions') + // Postgres folds unquoted identifiers to lowercase, so match column names case-insensitively + const hasColumn = (name) => Object.keys(tableDescription).some((column) => column.toLowerCase() === name.toLowerCase()) - if (!tableDescription.lastRefreshToken) { + if (!hasColumn('lastRefreshToken')) { logger.info(`${loggerPrefix} Adding lastRefreshToken column to sessions table`) await queryInterface.addColumn('sessions', 'lastRefreshToken', { type: queryInterface.sequelize.Sequelize.DataTypes.STRING, @@ -33,7 +35,7 @@ async function up({ context: { queryInterface, logger } }) { logger.info(`${loggerPrefix} lastRefreshToken column already exists in sessions table`) } - if (!tableDescription.lastRefreshTokenExpiresAt) { + if (!hasColumn('lastRefreshTokenExpiresAt')) { logger.info(`${loggerPrefix} Adding lastRefreshTokenExpiresAt column to sessions table`) await queryInterface.addColumn('sessions', 'lastRefreshTokenExpiresAt', { type: queryInterface.sequelize.Sequelize.DataTypes.DATE, @@ -60,15 +62,17 @@ async function down({ context: { queryInterface, logger } }) { if (await queryInterface.tableExists('sessions')) { const tableDescription = await queryInterface.describeTable('sessions') + // Postgres folds unquoted identifiers to lowercase, so match column names case-insensitively + const hasColumn = (name) => Object.keys(tableDescription).some((column) => column.toLowerCase() === name.toLowerCase()) - if (tableDescription.lastRefreshToken) { + if (hasColumn('lastRefreshToken')) { logger.info(`${loggerPrefix} Removing lastRefreshToken column from sessions table`) await queryInterface.removeColumn('sessions', 'lastRefreshToken') } else { logger.info(`${loggerPrefix} lastRefreshToken column does not exist in sessions table`) } - if (tableDescription.lastRefreshTokenExpiresAt) { + if (hasColumn('lastRefreshTokenExpiresAt')) { logger.info(`${loggerPrefix} Removing lastRefreshTokenExpiresAt column from sessions table`) await queryInterface.removeColumn('sessions', 'lastRefreshTokenExpiresAt') } else { diff --git a/test/server/migrations/v2.35.0-add-last-refresh-token.test.js b/test/server/migrations/v2.35.0-add-last-refresh-token.test.js new file mode 100644 index 000000000..7074c9577 --- /dev/null +++ b/test/server/migrations/v2.35.0-add-last-refresh-token.test.js @@ -0,0 +1,98 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const { up, down } = require('../../../server/migrations/v2.35.0-add-last-refresh-token') +const { Sequelize } = require('sequelize') +const Logger = require('../../../server/Logger') + +describe('migration-v2.35.0-add-last-refresh-token', () => { + let sequelize + let queryInterface + let loggerInfoStub + + beforeEach(() => { + sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + queryInterface = sequelize.getQueryInterface() + loggerInfoStub = sinon.stub(Logger, 'info') + }) + + afterEach(() => { + sinon.restore() + }) + + describe('up', () => { + beforeEach(async () => { + await queryInterface.createTable('sessions', { + id: { type: Sequelize.UUID, primaryKey: true }, + createdAt: { type: Sequelize.DATE, allowNull: false }, + updatedAt: { type: Sequelize.DATE, allowNull: false } + }) + }) + + it('should add lastRefreshToken columns when they do not exist', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const tableDescription = await queryInterface.describeTable('sessions') + expect(tableDescription.lastRefreshToken).to.exist + expect(tableDescription.lastRefreshTokenExpiresAt).to.exist + }) + + it('should not fail when the columns already exist', async () => { + await up({ context: { queryInterface, logger: Logger } }) + await up({ context: { queryInterface, logger: Logger } }) + + const tableDescription = await queryInterface.describeTable('sessions') + expect(tableDescription.lastRefreshToken).to.exist + expect(tableDescription.lastRefreshTokenExpiresAt).to.exist + expect(loggerInfoStub.calledWithMatch('lastRefreshToken column already exists')).to.equal(true) + }) + + it('should detect existing columns case-insensitively for postgres identifier folding', async () => { + // Postgres folds unquoted identifiers to lowercase, so describeTable on a + // migrated postgres database returns lowercase column names + const fakeQueryInterface = { + sequelize, + tableExists: async () => true, + describeTable: async () => ({ + id: {}, + lastrefreshtoken: {}, + lastrefreshtokenexpiresat: {} + }), + addColumn: async () => { + throw new Error('addColumn must not be called for existing lowercase columns') + } + } + + await up({ context: { queryInterface: fakeQueryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWithMatch('lastRefreshToken column already exists')).to.equal(true) + expect(loggerInfoStub.calledWithMatch('lastRefreshTokenExpiresAt column already exists')).to.equal(true) + }) + }) + + describe('down', () => { + beforeEach(async () => { + await queryInterface.createTable('sessions', { + id: { type: Sequelize.UUID, primaryKey: true }, + lastRefreshToken: { type: Sequelize.STRING, allowNull: true }, + lastRefreshTokenExpiresAt: { type: Sequelize.DATE, allowNull: true }, + createdAt: { type: Sequelize.DATE, allowNull: false }, + updatedAt: { type: Sequelize.DATE, allowNull: false } + }) + }) + + it('should remove lastRefreshToken columns when they exist', async () => { + await down({ context: { queryInterface, logger: Logger } }) + + const tableDescription = await queryInterface.describeTable('sessions') + expect(tableDescription.lastRefreshToken).to.not.exist + expect(tableDescription.lastRefreshTokenExpiresAt).to.not.exist + }) + + it('should not fail when the columns do not exist', async () => { + await down({ context: { queryInterface, logger: Logger } }) + await down({ context: { queryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWithMatch('lastRefreshToken column does not exist')).to.equal(true) + }) + }) +}) From 57fcda5b03ca82d036b879ecc1933c2448237f1b Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 2 Aug 2026 18:39:42 -0400 Subject: [PATCH 36/38] fix user cache misses for mixed-case username and email lookups getUserByUsername/getUserByEmail normalize to lowercase before the cache lookup, but the cache comparators matched case-sensitively, so a mixed-case stored username never cache-hit and every login ran an unindexed LOWER() scan. Compare case-insensitively inside the cache. --- server/models/User.js | 8 ++++++-- test/server/models/User.test.js | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/server/models/User.js b/server/models/User.js index f71f80f3b..d93ebc02e 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -18,12 +18,16 @@ class UserCache { } getByEmail(email) { - const user = this.cache.find((u) => u.email === email) + if (!email) return null + const normalizedEmail = email.toLowerCase() + const user = this.cache.find((u) => u.email && u.email.toLowerCase() === normalizedEmail) return user } getByUsername(username) { - const user = this.cache.find((u) => u.username === username) + if (!username) return null + const normalizedUsername = username.toLowerCase() + const user = this.cache.find((u) => u.username && u.username.toLowerCase() === normalizedUsername) return user } diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js index 734c31609..be317eb44 100644 --- a/test/server/models/User.test.js +++ b/test/server/models/User.test.js @@ -43,6 +43,38 @@ describe('User model', () => { expect(options.where.attribute.args[0].col).to.equal('email') expect(options.where.logic).to.equal('example.user@example.com') }) + + it('should hit the user cache for mixed-case username lookups', async () => { + User.sequelize = { + models: { + mediaProgress: {} + } + } + const cachedUser = { id: 'cache-test-user-1', username: 'CacheTestUser', email: 'cachetest@example.com', extraData: {} } + const findOneStub = sinon.stub(User, 'findOne').resolves(cachedUser) + + await User.getUserByUsername('CacheTestUser') + await User.getUserByUsername('cachetestuser') + await User.getUserByUsername('CACHETESTUSER') + + expect(findOneStub.callCount).to.equal(1) + }) + + it('should hit the user cache for mixed-case email lookups', async () => { + User.sequelize = { + models: { + mediaProgress: {} + } + } + const cachedUser = { id: 'cache-test-user-2', username: 'CacheEmailUser', email: 'CacheMail@Example.com', extraData: {} } + const findOneStub = sinon.stub(User, 'findOne').resolves(cachedUser) + + await User.getUserByEmail('CacheMail@Example.com') + await User.getUserByEmail('cachemail@example.com') + await User.getUserByEmail('CACHEMAIL@EXAMPLE.COM') + + expect(findOneStub.callCount).to.equal(1) + }) }) describe('getUserByIdOrOldId', () => { From 3f2919fce50c05dfd21415f594a02f8bbc149a53 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 2 Aug 2026 18:39:42 -0400 Subject: [PATCH 37/38] limit shelf query fallback to the postgres dialect withShelfFallback turned any shelf error into an empty shelf on both dialects, hiding real bugs and changing sqlite behavior versus upstream. Rethrow on sqlite and keep the empty-shelf fallback only for postgres, where dialect edge cases were the original motivation. --- server/utils/queries/libraryFilters.js | 2 ++ .../utils/queries/libraryFilters.test.js | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index fd261e050..501f92dc0 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -14,6 +14,8 @@ async function withShelfFallback(scope, fallbackValue, action) { try { return await action() } catch (error) { + // Only postgres shelves fall back to empty results - on sqlite keep upstream behavior of surfacing the error + if (!Database.isPostgresDialect()) throw error Logger.error(`[LibraryFilters] Failed to load ${scope}`, error) return fallbackValue } diff --git a/test/server/utils/queries/libraryFilters.test.js b/test/server/utils/queries/libraryFilters.test.js index 1c9a2836f..f956997ce 100644 --- a/test/server/utils/queries/libraryFilters.test.js +++ b/test/server/utils/queries/libraryFilters.test.js @@ -1,6 +1,7 @@ const { expect } = require('chai') const sinon = require('sinon') +const Database = require('../../../../server/Database') const Logger = require('../../../../server/Logger') const libraryFilters = require('../../../../server/utils/queries/libraryFilters') const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters') @@ -12,6 +13,7 @@ describe('libraryFilters shelf resilience', () => { }) it('should return empty discover shelf when discover query fails', async () => { + sinon.stub(Database, 'isPostgresDialect').returns(true) sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').rejects(new Error('discover failed')) const errorStub = sinon.stub(Logger, 'error') @@ -22,6 +24,7 @@ describe('libraryFilters shelf resilience', () => { }) it('should return empty in-progress shelf when a query fails', async () => { + sinon.stub(Database, 'isPostgresDialect').returns(true) sinon.stub(libraryItemsBookFilters, 'getFilteredLibraryItems').rejects(new Error('progress failed')) const errorStub = sinon.stub(Logger, 'error') @@ -32,6 +35,7 @@ describe('libraryFilters shelf resilience', () => { }) it('should return empty continue-series shelf when a query fails', async () => { + sinon.stub(Database, 'isPostgresDialect').returns(true) sinon.stub(libraryItemsBookFilters, 'getContinueSeriesLibraryItems').rejects(new Error('continue failed')) const errorStub = sinon.stub(Logger, 'error') @@ -42,6 +46,7 @@ describe('libraryFilters shelf resilience', () => { }) it('should return empty newest podcast episodes shelf when a query fails', async () => { + sinon.stub(Database, 'isPostgresDialect').returns(true) sinon.stub(libraryItemsPodcastFilters, 'getFilteredPodcastEpisodes').rejects(new Error('podcast failed')) const errorStub = sinon.stub(Logger, 'error') @@ -51,6 +56,21 @@ describe('libraryFilters shelf resilience', () => { expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load newest-podcast-episodes shelf for library "library-1"')).to.equal(true) }) + it('should rethrow shelf query errors on sqlite to keep upstream behavior', async () => { + sinon.stub(Database, 'isPostgresDialect').returns(false) + sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').rejects(new Error('discover failed')) + + let error + try { + await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10) + } catch (caughtError) { + error = caughtError + } + + expect(error).to.be.an('error') + expect(error.message).to.equal('discover failed') + }) + it('should keep discover shelf mapping behavior when query succeeds', async () => { const libraryItem = { toOldJSONMinified: () => ({ id: 'item-1' }), From 49d0a66c1de64366bd4544a538144faba7ccc763 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 2 Aug 2026 18:58:42 -0400 Subject: [PATCH 38/38] make v2.33.0 migration index checks work on postgres Sequelize showIndex matches the table name case-sensitively, but postgres folds unquoted identifiers to lowercase, so existing indexes were never detected and addIndex failed with already-exists errors, crash-looping the server on startup. Query pg_indexes with the folded table name on postgres and compare index names case-insensitively. --- .../v2.33.0-add-discover-query-indexes.js | 25 +++- ...v2.33.0-add-discover-query-indexes.test.js | 118 ++++++++++++++++++ 2 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 test/server/migrations/v2.33.0-add-discover-query-indexes.test.js diff --git a/server/migrations/v2.33.0-add-discover-query-indexes.js b/server/migrations/v2.33.0-add-discover-query-indexes.js index ebd92bbaa..8500a5c18 100644 --- a/server/migrations/v2.33.0-add-discover-query-indexes.js +++ b/server/migrations/v2.33.0-add-discover-query-indexes.js @@ -45,8 +45,8 @@ async function down({ context: { queryInterface, logger } }) { } async function addIndexIfMissing(queryInterface, logger, index) { - const existing = await queryInterface.showIndex(index.table) - if (existing.some((i) => i.name === index.name)) { + const existing = await showIndexNames(queryInterface, index.table) + if (existing.some((name) => name.toLowerCase() === index.name.toLowerCase())) { logger.info(`${loggerPrefix} index ${index.name} already exists on ${index.table}`) return } @@ -60,8 +60,8 @@ async function addIndexIfMissing(queryInterface, logger, index) { } async function removeIndexIfExists(queryInterface, logger, index) { - const existing = await queryInterface.showIndex(index.table) - if (!existing.some((i) => i.name === index.name)) { + const existing = await showIndexNames(queryInterface, index.table) + if (!existing.some((name) => name.toLowerCase() === index.name.toLowerCase())) { logger.info(`${loggerPrefix} index ${index.name} does not exist on ${index.table}`) return } @@ -71,4 +71,21 @@ async function removeIndexIfExists(queryInterface, logger, index) { logger.info(`${loggerPrefix} removed index ${index.name}`) } +/** + * Sequelize showIndex matches the table name case-sensitively, but postgres folds + * unquoted identifiers to lowercase, so query pg_indexes directly on postgres. + * + * @returns {Promise} index names on the table + */ +async function showIndexNames(queryInterface, table) { + if (queryInterface.sequelize.getDialect() === 'postgres') { + const [rows] = await queryInterface.sequelize.query('SELECT indexname AS name FROM pg_indexes WHERE schemaname = current_schema() AND tablename = $1', { + bind: [table.toLowerCase()] + }) + return rows.map((row) => row.name) + } + const existing = await queryInterface.showIndex(table) + return existing.map((row) => row.name) +} + module.exports = { up, down } diff --git a/test/server/migrations/v2.33.0-add-discover-query-indexes.test.js b/test/server/migrations/v2.33.0-add-discover-query-indexes.test.js new file mode 100644 index 000000000..57d15675a --- /dev/null +++ b/test/server/migrations/v2.33.0-add-discover-query-indexes.test.js @@ -0,0 +1,118 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const { up, down } = require('../../../server/migrations/v2.33.0-add-discover-query-indexes') +const { Sequelize } = require('sequelize') +const Logger = require('../../../server/Logger') + +describe('migration-v2.33.0-add-discover-query-indexes', () => { + let sequelize + let queryInterface + let loggerInfoStub + + beforeEach(() => { + sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + queryInterface = sequelize.getQueryInterface() + loggerInfoStub = sinon.stub(Logger, 'info') + }) + + afterEach(() => { + sinon.restore() + }) + + describe('up', () => { + beforeEach(async () => { + await queryInterface.createTable('mediaProgresses', { + id: { type: Sequelize.UUID, primaryKey: true }, + userId: { type: Sequelize.UUID, allowNull: false }, + mediaItemId: { type: Sequelize.UUID, allowNull: false }, + isFinished: { type: Sequelize.BOOLEAN, allowNull: false }, + currentTime: { type: Sequelize.FLOAT, allowNull: false } + }) + await queryInterface.createTable('bookSeries', { + id: { type: Sequelize.UUID, primaryKey: true }, + seriesId: { type: Sequelize.UUID, allowNull: false }, + bookId: { type: Sequelize.UUID, allowNull: false } + }) + }) + + it('should add both discover query indexes', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const mediaProgressIndexes = await queryInterface.showIndex('mediaProgresses') + expect(mediaProgressIndexes.some((i) => i.name === 'media_progresses_user_item_finished_time')).to.equal(true) + const bookSeriesIndexes = await queryInterface.showIndex('bookSeries') + expect(bookSeriesIndexes.some((i) => i.name === 'book_series_series_book')).to.equal(true) + }) + + it('should not fail when the indexes already exist', async () => { + await up({ context: { queryInterface, logger: Logger } }) + await up({ context: { queryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWithMatch('index media_progresses_user_item_finished_time already exists')).to.equal(true) + expect(loggerInfoStub.calledWithMatch('index book_series_series_book already exists')).to.equal(true) + }) + + it('should detect existing indexes on postgres via pg_indexes with folded table names', async () => { + // Sequelize showIndex matches relname case-sensitively and misses folded + // lowercase postgres tables, so the migration must query pg_indexes instead + const queries = [] + const fakeQueryInterface = { + sequelize: { + getDialect: () => 'postgres', + query: async (sql, options) => { + queries.push({ sql, options }) + return [[{ name: 'media_progresses_user_item_finished_time' }, { name: 'book_series_series_book' }]] + } + }, + addIndex: async () => { + throw new Error('addIndex must not be called for existing indexes') + }, + showIndex: async () => { + throw new Error('showIndex must not be used on postgres') + } + } + + await up({ context: { queryInterface: fakeQueryInterface, logger: Logger } }) + + expect(queries.length).to.be.greaterThan(0) + expect(queries.every((q) => q.sql.includes('pg_indexes'))).to.equal(true) + expect(queries.map((q) => q.options.bind[0])).to.include.members(['mediaprogresses', 'bookseries']) + expect(loggerInfoStub.calledWithMatch('index media_progresses_user_item_finished_time already exists')).to.equal(true) + expect(loggerInfoStub.calledWithMatch('index book_series_series_book already exists')).to.equal(true) + }) + }) + + describe('down', () => { + beforeEach(async () => { + await queryInterface.createTable('mediaProgresses', { + id: { type: Sequelize.UUID, primaryKey: true }, + userId: { type: Sequelize.UUID, allowNull: false }, + mediaItemId: { type: Sequelize.UUID, allowNull: false }, + isFinished: { type: Sequelize.BOOLEAN, allowNull: false }, + currentTime: { type: Sequelize.FLOAT, allowNull: false } + }) + await queryInterface.createTable('bookSeries', { + id: { type: Sequelize.UUID, primaryKey: true }, + seriesId: { type: Sequelize.UUID, allowNull: false }, + bookId: { type: Sequelize.UUID, allowNull: false } + }) + await up({ context: { queryInterface, logger: Logger } }) + }) + + it('should remove both discover query indexes', async () => { + await down({ context: { queryInterface, logger: Logger } }) + + const mediaProgressIndexes = await queryInterface.showIndex('mediaProgresses') + expect(mediaProgressIndexes.some((i) => i.name === 'media_progresses_user_item_finished_time')).to.equal(false) + const bookSeriesIndexes = await queryInterface.showIndex('bookSeries') + expect(bookSeriesIndexes.some((i) => i.name === 'book_series_series_book')).to.equal(false) + }) + + it('should not fail when the indexes do not exist', async () => { + await down({ context: { queryInterface, logger: Logger } }) + await down({ context: { queryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWithMatch('index media_progresses_user_item_finished_time does not exist')).to.equal(true) + }) + }) +})