From 453cd2587bf014e9b799deab593d4c8d9dee7347 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 16:52:24 -0500 Subject: [PATCH 01/19] add postgres dialect support for homelab migration --- package-lock.json | 171 +++++++++++- package.json | 3 + server/Database.js | 152 ++++++++--- server/Server.js | 25 +- server/controllers/LibraryController.js | 16 +- server/managers/MigrationManager.js | 52 ++-- .../v2.20.0-improve-author-sort-queries.js | 10 + server/models/Author.js | 9 +- server/models/Book.js | 9 +- server/models/LibraryItem.js | 13 +- server/models/Series.js | 9 +- server/scanner/LibraryScanner.js | 7 +- server/scripts/migrateSqliteToPostgres.js | 243 ++++++++++++++++++ server/utils/queries/libraryFilters.js | 7 +- server/utils/queries/libraryItemFilters.js | 11 +- .../utils/queries/libraryItemsBookFilters.js | 94 +++++-- .../queries/libraryItemsPodcastFilters.js | 25 +- server/utils/queries/seriesFilters.js | 21 +- server/utils/sqlDialectHelpers.js | 75 ++++++ test/server/utils/sqlDialectHelpers.test.js | 58 +++++ 20 files changed, 851 insertions(+), 159 deletions(-) create mode 100644 server/scripts/migrateSqliteToPostgres.js create mode 100644 server/utils/sqlDialectHelpers.js create mode 100644 test/server/utils/sqlDialectHelpers.test.js diff --git a/package-lock.json b/package-lock.json index 08707893d..178e19626 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "audiobookshelf", - "version": "2.32.1", + "version": "2.32.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "audiobookshelf", - "version": "2.32.1", + "version": "2.32.2", "license": "GPL-3.0", "dependencies": { "axios": "^0.27.2", @@ -23,6 +23,8 @@ "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", + "pg": "^8.19.0", + "pg-hstore": "^2.3.4", "semver": "^7.6.3", "sequelize": "^6.35.2", "socket.io": "^4.5.4", @@ -4138,10 +4140,106 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" }, + "node_modules/pg": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.19.0.tgz", + "integrity": "sha512-QIcLGi508BAHkQ3pJNptsFz5WQMlpGbuBGBaIaXsWK8mel2kQ/rThYI+DbgjUvZrIr7MiuEuc9LcChJoEZK1xQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.11.0", + "pg-pool": "^3.12.0", + "pg-protocol": "^1.12.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, "node_modules/pg-connection-string": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", - "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", + "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", + "license": "MIT" + }, + "node_modules/pg-hstore": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz", + "integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==", + "license": "MIT", + "dependencies": { + "underscore": "^1.13.1" + }, + "engines": { + "node": ">= 0.8.x" + } + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.12.0.tgz", + "integrity": "sha512-eIJ0DES8BLaziFHW7VgJEBPi5hg3Nyng5iKpYtj3wbcAUV9A1wLgWiY7ajf/f/oO1wfxt83phXPY8Emztg7ITg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.12.0.tgz", + "integrity": "sha512-uOANXNRACNdElMXJ0tPz6RBM0XQ61nONGAwlt8da5zs/iUOOCLBQOHSXnrC6fMsvtjxbOJrZZl5IScGv+7mpbg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } }, "node_modules/picocolors": { "version": "1.0.0", @@ -4173,6 +4271,45 @@ "node": ">=8" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -4946,6 +5083,15 @@ "node": ">=8" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -5265,6 +5411,12 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "node_modules/underscore": { + "version": "1.13.8", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", + "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", + "license": "MIT" + }, "node_modules/unique-filename": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", @@ -5476,6 +5628,15 @@ "node": ">=4.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 3ee3fb391..87697288c 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "docker": "docker buildx build --platform linux/amd64,linux/arm64 --push . -t advplyr/audiobookshelf", "docker-amd64-local": "docker buildx build --platform linux/amd64 --load . -t advplyr/audiobookshelf-amd64-local", "docker-arm64-local": "docker buildx build --platform linux/arm64 --load . -t advplyr/audiobookshelf-arm64-local", + "migrate-sqlite-to-postgres": "node server/scripts/migrateSqliteToPostgres.js", "deploy-linux": "node deploy/linux", "test": "mocha", "coverage": "nyc mocha" @@ -52,6 +53,8 @@ "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", + "pg": "^8.19.0", + "pg-hstore": "^2.3.4", "semver": "^7.6.3", "sequelize": "^6.35.2", "socket.io": "^4.5.4", diff --git a/server/Database.js b/server/Database.js index 213c2c61b..68f6e2729 100644 --- a/server/Database.js +++ b/server/Database.js @@ -14,6 +14,7 @@ class Database { constructor() { this.sequelize = null this.dbPath = null + this.dialect = 'sqlite' this.isNew = false // New absdatabase.sqlite created this.hasRootUser = false // Used to show initialization page in web ui @@ -162,11 +163,37 @@ class Database { return this.models.device } + /** + * @returns {'sqlite'|'postgres'} + */ + getConfiguredDialect() { + const explicitDialect = process.env.DB_DIALECT?.trim()?.toLowerCase() + if (explicitDialect === 'postgres' || explicitDialect === 'sqlite') { + return explicitDialect + } + + const databaseUrl = process.env.DATABASE_URL || '' + if (databaseUrl.startsWith('postgres://') || databaseUrl.startsWith('postgresql://')) { + return 'postgres' + } + + return 'sqlite' + } + + isSqliteDialect() { + return this.dialect === 'sqlite' + } + + isPostgresDialect() { + return this.dialect === 'postgres' + } + /** * Check if db file exists * @returns {boolean} */ async checkHasDb() { + if (!this.isSqliteDialect()) return true if (!(await fs.pathExists(this.dbPath))) { Logger.info(`[Database] absdatabase.sqlite not found at ${this.dbPath}`) return false @@ -174,20 +201,39 @@ class Database { return true } + /** + * Check if any user tables exist (for networked dialects) + * @returns {Promise} + */ + async checkHasTables() { + const queryInterface = this.sequelize.getQueryInterface() + const tables = await queryInterface.showAllTables() + return Array.isArray(tables) && tables.length > 0 + } + /** * Connect to db, build models and run migrations * @param {boolean} [force=false] Used for testing, drops & re-creates all tables */ async init(force = false) { - this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') - - // First check if this is a new database - this.isNew = !(await this.checkHasDb()) || force + this.dialect = this.getConfiguredDialect() + if (this.isSqliteDialect()) { + this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') + // First check if this is a new database + this.isNew = !(await this.checkHasDb()) || force + } else { + this.dbPath = process.env.DATABASE_URL || null + this.isNew = !!force + } if (!(await this.connect())) { throw new Error('Database connection failed') } + if (this.isPostgresDialect() && !force) { + this.isNew = !(await this.checkHasTables()) + } + try { const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath) await migrationManager.init(packageJson.version) @@ -214,7 +260,11 @@ class Database { * @returns {boolean} */ async connect() { - Logger.info(`[Database] Initializing db at "${this.dbPath}"`) + if (this.isSqliteDialect()) { + Logger.info(`[Database] Initializing sqlite db at "${this.dbPath}"`) + } else { + Logger.info(`[Database] Initializing postgres db connection`) + } let logging = false let benchmark = false @@ -229,13 +279,26 @@ class Database { benchmark = true } - this.sequelize = new Sequelize({ - dialect: 'sqlite', - storage: this.dbPath, - logging: logging, - benchmark: benchmark, - transactionType: 'IMMEDIATE' - }) + if (this.isSqliteDialect()) { + this.sequelize = new Sequelize({ + dialect: 'sqlite', + storage: this.dbPath, + logging: logging, + benchmark: benchmark, + transactionType: 'IMMEDIATE' + }) + } else { + if (!process.env.DATABASE_URL) { + Logger.error(`[Database] DATABASE_URL is required when DB_DIALECT=postgres`) + return false + } + + this.sequelize = new Sequelize(process.env.DATABASE_URL, { + dialect: 'postgres', + logging: logging, + benchmark: benchmark + }) + } // Helper function this.sequelize.uppercaseFirst = (str) => (str ? `${str[0].toUpperCase()}${str.substr(1)}` : '') @@ -243,34 +306,36 @@ class Database { try { await this.sequelize.authenticate() - // Set SQLite pragmas from environment variables - const allowedPragmas = [ - { name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' }, - { name: 'cache_size', env: 'SQLITE_CACHE_SIZE' }, - { name: 'temp_store', env: 'SQLITE_TEMP_STORE' } - ] + if (this.isSqliteDialect()) { + // Set SQLite pragmas from environment variables + const allowedPragmas = [ + { name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' }, + { name: 'cache_size', env: 'SQLITE_CACHE_SIZE' }, + { name: 'temp_store', env: 'SQLITE_TEMP_STORE' } + ] - for (const pragma of allowedPragmas) { - const value = process.env[pragma.env] - if (value !== undefined) { - try { - Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`) - await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`) - const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`) - Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result) - } catch (error) { - Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error) + for (const pragma of allowedPragmas) { + const value = process.env[pragma.env] + if (value !== undefined) { + try { + Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`) + await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`) + const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`) + Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result) + } catch (error) { + Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error) + } } } - } - if (process.env.NUSQLITE3_PATH) { - await this.loadExtension(process.env.NUSQLITE3_PATH) - Logger.info(`[Database] Db supports unaccent and unicode foldings`) - this.supportsUnaccent = true - this.supportsUnicodeFoldings = true + if (process.env.NUSQLITE3_PATH) { + await this.loadExtension(process.env.NUSQLITE3_PATH) + Logger.info(`[Database] Db supports unaccent and unicode foldings`) + this.supportsUnaccent = true + this.supportsUnicodeFoldings = true + } } - Logger.info(`[Database] Db connection was successful`) + Logger.info(`[Database] Db connection was successful (${this.dialect})`) return true } catch (error) { Logger.error(`[Database] Failed to connect to db`, error) @@ -307,7 +372,7 @@ class Database { * Disconnect from db */ async disconnect() { - Logger.info(`[Database] Disconnecting sqlite db`) + Logger.info(`[Database] Disconnecting ${this.dialect} db`) await this.sequelize.close() } @@ -315,7 +380,7 @@ class Database { * Reconnect to db and init */ async reconnect() { - Logger.info(`[Database] Reconnecting sqlite db`) + Logger.info(`[Database] Reconnecting ${this.dialect} db`) await this.init() } @@ -368,7 +433,7 @@ class Database { * Loads most of the data from the database. This is a temporary solution. */ async loadData() { - if (this.isNew && (await dbMigration.checkShouldMigrate())) { + if (this.isSqliteDialect() && this.isNew && (await dbMigration.checkShouldMigrate())) { Logger.info(`[Database] New database was created and old database was detected - migrating old to new`) await dbMigration.migrate(this.models) } @@ -847,6 +912,11 @@ WHERE EXISTS ( * It adds triggers to update libraryItems.title[IgnorePrefix] when (books|podcasts).title[IgnorePrefix] is updated */ async addTriggers() { + if (!this.isSqliteDialect()) { + Logger.info(`[Database] Skipping sqlite-only triggers for dialect ${this.dialect}`) + return + } + await this.addTriggerIfNotExists('books', 'title', 'id', 'libraryItems', 'title', 'mediaId') await this.addTriggerIfNotExists('books', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId') await this.addTriggerIfNotExists('podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId') @@ -954,6 +1024,7 @@ WHERE EXISTS ( this.supportsUnaccent = supportsUnaccent this.query = query this.hasAccents = false + this.dialect = sequelize.getDialect() } /** @@ -989,9 +1060,10 @@ WHERE EXISTS ( */ matchExpression(column) { const pattern = this.sequelize.escape(`%${this.query}%`) - if (!this.supportsUnaccent) return `${column} LIKE ${pattern}` + const likeOperator = this.dialect === 'postgres' ? 'ILIKE' : 'LIKE' + if (!this.supportsUnaccent) return `${column} ${likeOperator} ${pattern}` const normalizedColumn = this.hasAccents ? column : this.normalize(column) - return `${normalizedColumn} LIKE ${pattern}` + return `${normalizedColumn} ${likeOperator} ${pattern}` } } } diff --git a/server/Server.js b/server/Server.js index d6f748a1e..7d51041a8 100644 --- a/server/Server.js +++ b/server/Server.js @@ -477,14 +477,27 @@ class Server { // Remove series from hide from continue listening that no longer exist try { - const users = await Database.sequelize.query(`SELECT u.id, u.username, u.extraData, json_group_array(value) AS seriesIdsToRemove FROM users u, json_each(u.extraData->"seriesHideFromContinueListening") LEFT JOIN series se ON se.id = value WHERE se.id IS NULL GROUP BY u.id;`, { - model: Database.userModel, - type: Sequelize.QueryTypes.SELECT + const users = await Database.userModel.findAll({ + attributes: ['id', 'username', 'extraData'] }) + for (const user of users) { - const extraData = JSON.parse(user.extraData) - const existingSeriesIds = extraData.seriesHideFromContinueListening - const seriesIdsToRemove = JSON.parse(user.dataValues.seriesIdsToRemove) + const extraData = typeof user.extraData === 'string' ? JSON.parse(user.extraData || '{}') : user.extraData || {} + const existingSeriesIds = Array.isArray(extraData.seriesHideFromContinueListening) ? extraData.seriesHideFromContinueListening : [] + if (!existingSeriesIds.length) continue + + const existingSeries = await Database.seriesModel.findAll({ + attributes: ['id'], + where: { + id: { + [Sequelize.Op.in]: existingSeriesIds + } + } + }) + const existingSeriesSet = new Set(existingSeries.map((series) => series.id)) + const seriesIdsToRemove = existingSeriesIds.filter((seriesId) => !existingSeriesSet.has(seriesId)) + if (!seriesIdsToRemove.length) continue + Logger.info(`[Server] Found ${seriesIdsToRemove.length} non-existent series in seriesHideFromContinueListening for user "${user.username}" - Removing (${seriesIdsToRemove.join(',')})`) const newExtraData = { ...extraData, diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 55ef45690..104d21467 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -24,6 +24,7 @@ const libraryFilters = require('../utils/queries/libraryFilters') const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters') const authorFilters = require('../utils/queries/authorFilters') const zipHelpers = require('../utils/zipHelpers') +const { isPostgres, noCaseSortExpression, jsonArrayExpand } = require('../utils/sqlDialectHelpers') /** * @typedef RequestUserObject @@ -1039,9 +1040,9 @@ class LibraryController { let order = undefined const direction = payload.sortDesc ? 'DESC' : 'ASC' if (payload.sortBy === 'name') { - order = [[Sequelize.literal('name COLLATE NOCASE'), direction]] + order = [[Sequelize.literal(noCaseSortExpression('name', Database.sequelize)), direction]] } else if (payload.sortBy === 'lastFirst') { - order = [[Sequelize.literal('lastFirst COLLATE NOCASE'), direction]] + order = [[Sequelize.literal(noCaseSortExpression('lastFirst', Database.sequelize)), direction]] } else if (payload.sortBy === 'addedAt') { order = [['createdAt', direction]] } else if (payload.sortBy === 'updatedAt') { @@ -1338,16 +1339,23 @@ class LibraryController { const fileExt = req.query.ext === 'abs' ? 'abs' : 'json' const metadataFilename = `metadata.${fileExt}` + const metadataFilenameCountQuery = isPostgres(Database.sequelize) + ? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{metadata,filename}' = :metadataFilename)` + : `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = :metadataFilename)` + const libraryItemsWithMetadata = await Database.libraryItemModel.findAll({ attributes: ['id', 'libraryFiles'], where: [ { libraryId: req.library.id }, - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = "${metadataFilename}")`), { + Sequelize.where(Sequelize.literal(metadataFilenameCountQuery), { [Sequelize.Op.gte]: 1 }) - ] + ], + replacements: { + metadataFilename + } }) if (!libraryItemsWithMetadata.length) { Logger.info(`[LibraryController] No ${metadataFilename} files found to remove`) diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index e302038ff..588771318 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -94,20 +94,26 @@ class MigrationManager { // Only proceed with migration if there are migrations to run if (migrationsToRun.length > 0) { + const dialect = typeof this.sequelize.getDialect === 'function' ? this.sequelize.getDialect() : 'sqlite' + const isSqlite = !dialect || dialect === 'sqlite' const originalDbPath = path.join(this.configPath, 'absdatabase.sqlite') const backupDbPath = path.join(this.configPath, 'absdatabase.backup.sqlite') try { Logger.info(`[MigrationManager] Migrating database ${migrationDirection} to version ${this.serverVersion}`) Logger.info(`[MigrationManager] Migrations to run: ${migrationsToRun.join(', ')}`) - // Create a backup copy of the SQLite database before starting migrations - await fs.copy(originalDbPath, backupDbPath) - Logger.info('Created a backup of the original database.') + if (isSqlite) { + // Create a backup copy of the SQLite database before starting migrations + await fs.copy(originalDbPath, backupDbPath) + Logger.info('Created a backup of the original database.') + } // Run migrations await this.umzug[migrationDirection]({ migrations: migrationsToRun, rerun: 'ALLOW' }) - // Clean up the backup - await fs.remove(backupDbPath) + if (isSqlite) { + // Clean up the backup + await fs.remove(backupDbPath) + } Logger.info('[MigrationManager] Migrations successfully applied to the original database.') } catch (error) { @@ -115,13 +121,15 @@ class MigrationManager { await this.sequelize.close() - // Step 3: If migration fails, save the failed original and restore the backup - const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite') - await fs.move(originalDbPath, failedDbPath, { overwrite: true }) - Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.') + if (isSqlite) { + // Step 3: If migration fails, save the failed original and restore the backup + const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite') + await fs.move(originalDbPath, failedDbPath, { overwrite: true }) + Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.') - await fs.move(backupDbPath, originalDbPath, { overwrite: true }) - Logger.info('[MigrationManager] Restored the original database from the backup.') + await fs.move(backupDbPath, originalDbPath, { overwrite: true }) + Logger.info('[MigrationManager] Restored the original database from the backup.') + } Logger.info('[MigrationManager] Migration failed. Exiting Audiobookshelf with code 1.') process.exit(1) @@ -191,16 +199,15 @@ class MigrationManager { } async fetchVersionsFromDatabase() { + const migrationsMetaTable = `"${MigrationManager.MIGRATIONS_META_TABLE}"` await this.checkOrCreateMigrationsMetaTable() - const [{ version }] = await this.sequelize.query("SELECT value as version FROM :migrationsMeta WHERE key = 'version'", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [{ version }] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, { type: Sequelize.QueryTypes.SELECT }) this.databaseVersion = version - const [{ maxVersion }] = await this.sequelize.query("SELECT value as maxVersion FROM :migrationsMeta WHERE key = 'maxVersion'", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [{ maxVersion }] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, { type: Sequelize.QueryTypes.SELECT }) this.maxVersion = maxVersion @@ -212,8 +219,7 @@ class MigrationManager { // If the table exists, check that the `version` and `maxVersion` rows exist if (migrationsMetaTableExists) { - const [{ count }] = await this.sequelize.query("SELECT COUNT(*) as count FROM :migrationsMeta WHERE key IN ('version', 'maxVersion')", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM "${MigrationManager.MIGRATIONS_META_TABLE}" WHERE key IN ('version', 'maxVersion')`, { type: Sequelize.QueryTypes.SELECT }) if (count < 2) { @@ -241,8 +247,8 @@ class MigrationManager { allowNull: false } }) - await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", { - replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`INSERT INTO "${MigrationManager.MIGRATIONS_META_TABLE}" (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, { + replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0' }, type: Sequelize.QueryTypes.INSERT }) Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`) @@ -299,8 +305,8 @@ class MigrationManager { async updateMaxVersion() { try { - await this.sequelize.query("UPDATE :migrationsMeta SET value = :maxVersion WHERE key = 'maxVersion'", { - replacements: { maxVersion: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :maxVersion WHERE key = 'maxVersion'`, { + replacements: { maxVersion: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) } catch (error) { @@ -311,8 +317,8 @@ class MigrationManager { async updateDatabaseVersion() { try { - await this.sequelize.query("UPDATE :migrationsMeta SET value = :version WHERE key = 'version'", { - replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :version WHERE key = 'version'`, { + replacements: { version: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) } catch (error) { diff --git a/server/migrations/v2.20.0-improve-author-sort-queries.js b/server/migrations/v2.20.0-improve-author-sort-queries.js index 53016c709..c5c2b8f5a 100644 --- a/server/migrations/v2.20.0-improve-author-sort-queries.js +++ b/server/migrations/v2.20.0-improve-author-sort-queries.js @@ -39,6 +39,11 @@ const authorsJoin = `${authors} JOIN ${bookAuthors} ON ${authors}.id = ${bookAut * @returns {Promise} - A promise that resolves when the migration is complete. */ async function up({ context: { queryInterface, logger } }) { + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`) + return + } + const helper = new MigrationHelper(queryInterface, logger) // Upwards migration script @@ -72,6 +77,11 @@ async function up({ context: { queryInterface, logger } }) { * @returns {Promise} - A promise that resolves when the migration is complete. */ async function down({ context: { queryInterface, logger } }) { + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`) + return + } + // Downward migration script logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) diff --git a/server/models/Author.js b/server/models/Author.js index 287b66976..8185020df 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -128,6 +128,8 @@ class Author extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' } + super.init( { id: { @@ -146,12 +148,7 @@ class Author extends Model { modelName: 'author', indexes: [ { - fields: [ - { - name: 'name', - collate: 'NOCASE' - } - ] + fields: [nameIndexField] }, // { // fields: [{ diff --git a/server/models/Book.js b/server/models/Book.js index 96371f3a2..08982bb03 100644 --- a/server/models/Book.js +++ b/server/models/Book.js @@ -137,6 +137,8 @@ class Book extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' } + super.init( { id: { @@ -171,12 +173,7 @@ class Book extends Model { modelName: 'book', indexes: [ { - fields: [ - { - name: 'title', - collate: 'NOCASE' - } - ] + fields: [titleIndexField] }, // { // fields: [{ diff --git a/server/models/LibraryItem.js b/server/models/LibraryItem.js index 16a521615..22c8b60a2 100644 --- a/server/models/LibraryItem.js +++ b/server/models/LibraryItem.js @@ -662,6 +662,11 @@ class LibraryItem extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' } + const titleIgnorePrefixIndexField = sequelize.getDialect() === 'postgres' ? 'titleIgnorePrefix' : { name: 'titleIgnorePrefix', collate: 'NOCASE' } + const authorNamesFirstLastIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesFirstLast' : { name: 'authorNamesFirstLast', collate: 'NOCASE' } + const authorNamesLastFirstIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesLastFirst' : { name: 'authorNamesLastFirst', collate: 'NOCASE' } + super.init( { id: { @@ -710,16 +715,16 @@ class LibraryItem extends Model { fields: ['libraryId', 'mediaType', 'createdAt'] }, { - fields: ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', titleIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', titleIgnorePrefixIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', authorNamesFirstLastIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', authorNamesLastFirstIndexField] }, { fields: ['libraryId', 'mediaId', 'mediaType'] diff --git a/server/models/Series.js b/server/models/Series.js index 6ca288464..28450a010 100644 --- a/server/models/Series.js +++ b/server/models/Series.js @@ -87,6 +87,8 @@ class Series extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' } + super.init( { id: { @@ -103,12 +105,7 @@ class Series extends Model { modelName: 'series', indexes: [ { - fields: [ - { - name: 'name', - collate: 'NOCASE' - } - ] + fields: [nameIndexField] }, // { // fields: [{ diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 640c82d76..f0ef43fec 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -14,6 +14,7 @@ const LibraryItemScanner = require('./LibraryItemScanner') const LibraryScan = require('./LibraryScan') const LibraryItemScanData = require('./LibraryItemScanData') const Task = require('../objects/Task') +const { isPostgres, jsonArrayExpand } = require('../utils/sqlDialectHelpers') class LibraryScanner { constructor() { @@ -674,12 +675,16 @@ async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingle // check if it was moved from another folder by comparing the ino to the library files const ino = await fileUtils.getIno(fullPath) if (!ino) return null + const inodeMatchQuery = isPostgres(Database.sequelize) + ? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{ino}' = :inode)` + : `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)` + const existingLibraryItem = await Database.libraryItemModel.findOneExpanded( [ { libraryId: libraryId }, - sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), { + sequelize.where(sequelize.literal(inodeMatchQuery), { [sequelize.Op.gt]: 0 }) ], diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js new file mode 100644 index 000000000..60acc11e2 --- /dev/null +++ b/server/scripts/migrateSqliteToPostgres.js @@ -0,0 +1,243 @@ +#!/usr/bin/env node + +const sqlite3 = require('sqlite3') +const { Client } = require('pg') + +const SQLITE_PATH = process.env.SQLITE_PATH || '/config/absdatabase.sqlite' +const DATABASE_URL = process.env.DATABASE_URL +const PG_SCHEMA = process.env.PG_SCHEMA || 'public' +const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500) +const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true' + +if (!DATABASE_URL) { + console.error('[migrate] DATABASE_URL is required') + process.exit(1) +} + +const preferredOrder = [ + 'migrationsMeta', + 'SequelizeMeta', + 'settings', + 'users', + 'apiKeys', + 'sessions', + 'libraries', + 'libraryFolders', + 'authors', + 'series', + 'books', + 'podcasts', + 'podcastEpisodes', + 'libraryItems', + 'bookAuthors', + 'bookSeries', + 'collections', + 'collectionBooks', + 'playlists', + 'playlistMediaItems', + 'mediaProgresses', + 'devices', + 'playbackSessions', + 'feeds', + 'feedEpisodes', + 'mediaItemShares', + 'customMetadataProviders' +] + +function quoteIdent(identifier) { + return `"${String(identifier).replace(/"/g, '""')}"` +} + +function openSqlite(filePath) { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(filePath, sqlite3.OPEN_READONLY, (err) => { + if (err) return reject(err) + resolve(db) + }) + }) +} + +function sqliteAll(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => { + if (err) return reject(err) + resolve(rows) + }) + }) +} + +function sqliteGet(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.get(sql, params, (err, row) => { + if (err) return reject(err) + resolve(row) + }) + }) +} + +function normalizeBoolean(value) { + if (value === null || value === undefined) return null + if (typeof value === 'boolean') return value + if (typeof value === 'number') return value !== 0 + if (typeof value === 'string') { + const v = value.trim().toLowerCase() + return v === 'true' || v === '1' || v === 't' + } + return !!value +} + +function normalizeJson(value) { + if (value === null || value === undefined || value === '') return null + if (typeof value === 'object') return value + try { + return JSON.parse(value) + } catch (error) { + return value + } +} + +function convertValue(value, pgColumn) { + if (!pgColumn) return value + const dataType = pgColumn.data_type + const udtName = pgColumn.udt_name + + if (dataType === 'boolean') { + return normalizeBoolean(value) + } + + if (dataType === 'json' || dataType === 'jsonb' || udtName === 'json' || udtName === 'jsonb') { + return normalizeJson(value) + } + + return value +} + +async function main() { + console.log(`[migrate] sqlite source: ${SQLITE_PATH}`) + console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`) + console.log(`[migrate] dry run: ${DRY_RUN}`) + + const sqliteDb = await openSqlite(SQLITE_PATH) + const pg = new Client({ connectionString: DATABASE_URL }) + await pg.connect() + + try { + const sqliteTablesRows = await sqliteAll( + sqliteDb, + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + ) + const sqliteTables = sqliteTablesRows.map((row) => row.name) + + const pgTablesRows = await pg.query( + `SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`, + [PG_SCHEMA] + ) + const pgTables = new Set(pgTablesRows.rows.map((row) => row.table_name)) + + const tablesToMigrate = sqliteTables.filter((table) => pgTables.has(table)) + tablesToMigrate.sort((a, b) => { + const ai = preferredOrder.indexOf(a) + const bi = preferredOrder.indexOf(b) + if (ai === -1 && bi === -1) return a.localeCompare(b) + if (ai === -1) return 1 + if (bi === -1) return -1 + return ai - bi + }) + + if (!tablesToMigrate.length) { + throw new Error('No overlapping tables found between SQLite and PostgreSQL') + } + + console.log(`[migrate] tables to migrate: ${tablesToMigrate.join(', ')}`) + + const pgColumnsByTable = new Map() + for (const table of tablesToMigrate) { + const columnsResult = await pg.query( + `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, + [PG_SCHEMA, table] + ) + const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name, column])) + pgColumnsByTable.set(table, columnMap) + } + + if (!DRY_RUN) { + await pg.query('BEGIN') + await pg.query('SET session_replication_role = replica') + + const truncateList = tablesToMigrate.map((table) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`).join(', ') + await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`) + console.log('[migrate] truncated target tables') + } + + for (const table of tablesToMigrate) { + const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(table)}`) + const pgColumns = pgColumnsByTable.get(table) + const insertColumns = rows.length ? Object.keys(rows[0]).filter((column) => pgColumns.has(column)) : [] + + if (!rows.length || !insertColumns.length) { + console.log(`[migrate] ${table}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`) + continue + } + + if (!DRY_RUN) { + for (let offset = 0; offset < rows.length; offset += BATCH_SIZE) { + const batchRows = rows.slice(offset, offset + BATCH_SIZE) + const valuesSql = [] + const params = [] + let paramIndex = 1 + + for (const row of batchRows) { + const placeholders = [] + for (const column of insertColumns) { + const pgColumn = pgColumns.get(column) + params.push(convertValue(row[column], pgColumn)) + placeholders.push(`$${paramIndex++}`) + } + valuesSql.push(`(${placeholders.join(', ')})`) + } + + const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)} (${insertColumns.map(quoteIdent).join(', ')}) VALUES ${valuesSql.join(', ')}` + await pg.query(insertSql, params) + } + } + + console.log(`[migrate] ${table}: ${rows.length} rows`) + } + + if (!DRY_RUN) { + await pg.query('SET session_replication_role = DEFAULT') + await pg.query('COMMIT') + console.log('[migrate] migration transaction committed') + } + + const parity = [] + for (const table of tablesToMigrate) { + const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(table)}`) + const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`) + parity.push({ + table, + sqliteCount: Number(sqliteCountRow.count || 0), + postgresCount: Number(pgCountResult.rows[0].count || 0) + }) + } + + const mismatches = parity.filter((row) => row.sqliteCount !== row.postgresCount) + if (mismatches.length) { + console.error('[migrate] row-count mismatches detected:') + mismatches.forEach((row) => { + console.error(`[migrate] ${row.table}: sqlite=${row.sqliteCount} postgres=${row.postgresCount}`) + }) + process.exitCode = 2 + } else { + console.log('[migrate] parity check passed for all migrated tables') + } + } finally { + sqliteDb.close() + await pg.end() + } +} + +main().catch((error) => { + console.error('[migrate] failed:', error) + process.exit(1) +}) diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index 7312b9d5d..49f5381e1 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -5,6 +5,7 @@ const libraryItemsBookFilters = require('./libraryItemsBookFilters') const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters') const { createNewSortInstance } = require('../../libs/fastSort') const { profile } = require('../../utils/profiler') +const { booleanLiteral, jsonArrayContainsAny } = require('../sqlDialectHelpers') const naturalSort = createNewSortInstance({ comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare }) @@ -235,13 +236,13 @@ module.exports = { if (userPermissionBookWhere.bookWhere.length) { let attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id' if (!user.canAccessExplicitContent) { - attrQuery += ' AND b.explicit = 0' + attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}` } if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (user.permissions.selectedTagsNotAccessible) { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0` } else { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0` } } seriesWhere.push( diff --git a/server/utils/queries/libraryItemFilters.js b/server/utils/queries/libraryItemFilters.js index 7f95d0ecc..15c3dedb7 100644 --- a/server/utils/queries/libraryItemFilters.js +++ b/server/utils/queries/libraryItemFilters.js @@ -2,6 +2,7 @@ const Sequelize = require('sequelize') const Database = require('../../Database') const libraryItemsBookFilters = require('./libraryItemsBookFilters') const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters') +const { jsonArrayContainsAny } = require('../sqlDialectHelpers') module.exports = { /** @@ -12,7 +13,7 @@ module.exports = { async getAllLibraryItemsWithTags(tags) { const libraryItems = [] const booksWithTag = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -46,7 +47,7 @@ module.exports = { libraryItems.push(libraryItem) } const podcastsWithTag = await Database.podcastModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -77,7 +78,7 @@ module.exports = { async getAllLibraryItemsWithGenres(genres) { const libraryItems = [] const booksWithGenre = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -107,7 +108,7 @@ module.exports = { libraryItems.push(libraryItem) } const podcastsWithGenre = await Database.podcastModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -138,7 +139,7 @@ module.exports = { async getAllLibraryItemsWithNarrators(narrators) { const libraryItems = [] const booksWithGenre = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(narrators) WHERE json_valid(narrators) AND json_each.value IN (:narrators))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('narrators', 'narrators', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 7ae1dc866..029448529 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -6,6 +6,7 @@ const authorFilters = require('./authorFilters') const ShareManager = require('../../managers/ShareManager') const { profile } = require('../profiler') const stringifySequelizeQuery = require('../stringifySequelizeQuery') +const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand } = require('../sqlDialectHelpers') const countCache = new Map() module.exports = { @@ -27,10 +28,10 @@ module.exports = { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { replacements['userTagsSelected'] = user.permissions.itemTagsSelected if (user.permissions.selectedTagsNotAccessible) { - bookWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0)) + bookWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0)) } else { bookWhere.push( - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), { + Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) ) @@ -189,7 +190,7 @@ module.exports = { } else if (group === 'explicit') { mediaWhere['explicit'] = true } else if (['genres', 'tags', 'narrators'].includes(group)) { - mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), { + mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) replacements.filterValue = value @@ -256,9 +257,9 @@ module.exports = { const getTitleOrder = () => { if (global.ServerSettings.sortingIgnorePrefix) { - return [Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir] + return [Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir] } else { - return [Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir] + return [Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir] } } @@ -273,20 +274,23 @@ module.exports = { } else if (sortBy === 'media.duration') { return [['duration', dir]] } else if (sortBy === 'media.metadata.publishedYear') { - return [[Sequelize.literal(`CAST(\`book\`.\`publishedYear\` AS INTEGER)`), dir]] + return [[Sequelize.literal('CAST(book.publishedYear AS INTEGER)'), dir]] } else if (sortBy === 'media.metadata.authorNameLF') { // Sort by author name last first, secondary sort by title - return [[Sequelize.literal('`libraryItem`.`authorNamesLastFirst` COLLATE NOCASE'), dir], getTitleOrder()] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesLastFirst', Database.sequelize)), dir], getTitleOrder()] } else if (sortBy === 'media.metadata.authorName') { // Sort by author name first last, secondary sort by title - return [[Sequelize.literal('`libraryItem`.`authorNamesFirstLast` COLLATE NOCASE'), dir], getTitleOrder()] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesFirstLast', Database.sequelize)), dir], getTitleOrder()] } else if (sortBy === 'media.metadata.title') { if (collapseseries) { - return [[Sequelize.literal('display_title COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('display_title', Database.sequelize)), dir]] } return [getTitleOrder()] } else if (sortBy === 'sequence') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' + if (Database.sequelize.getDialect() === 'postgres') { + return [[Sequelize.literal(`CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ${nullDir}`)]] + } return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]] } else if (sortBy === 'progress') { return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]] @@ -326,7 +330,13 @@ module.exports = { required: true } ], - order: [Sequelize.literal('CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')] + order: [ + Sequelize.literal( + Database.sequelize.getDialect() === 'postgres' + ? 'CAST("books.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST' + : 'CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST' + ) + ] }) const bookSeriesToInclude = [] const booksToInclude = [] @@ -455,12 +465,28 @@ module.exports = { }) } else if (filterGroup === 'ebooks' && filterValue === 'supplementary') { // TODO: Temp workaround for filtering supplementary ebook - libraryItemWhere['libraryFiles'] = { - [Sequelize.Op.substring]: `"isSupplementary":true` + if (Database.sequelize.getDialect() === 'postgres') { + libraryItemWhere[Sequelize.Op.and] = [ + Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), { + [Sequelize.Op.like]: '%"isSupplementary":true%' + }) + ] + } else { + libraryItemWhere['libraryFiles'] = { + [Sequelize.Op.substring]: `"isSupplementary":true` + } } } else if (filterGroup === 'ebooks' && filterValue === 'no-supplementary') { - libraryItemWhere['libraryFiles'] = { - [Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`) + if (Database.sequelize.getDialect() === 'postgres') { + libraryItemWhere[Sequelize.Op.and] = [ + Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), { + [Sequelize.Op.notLike]: '%"isSupplementary":true%' + }) + ] + } else { + libraryItemWhere['libraryFiles'] = { + [Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`) + } } } else if (filterGroup === 'missing' && filterValue === 'authors') { authorInclude = { @@ -502,7 +528,13 @@ module.exports = { }) if (sortBy !== 'sequence') { // Secondary sort by sequence - sortOrder.push([Sequelize.literal('CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]) + sortOrder.push([ + Sequelize.literal( + Database.sequelize.getDialect() === 'postgres' + ? 'CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST' + : 'CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST' + ) + ]) } } else if (filterGroup === 'issues') { libraryItemWhere[Sequelize.Op.or] = [ @@ -594,10 +626,14 @@ module.exports = { // When collapsing series and sorting by title then use the series name instead of the book title // for this set an attribute "display_title" to use in sorting + const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."title"' : '`libraryItem`.`title`' + const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."titleIgnorePrefix"' : '`libraryItem`.`titleIgnorePrefix`' + const fallbackFn = coalesceFunctionName(Database.sequelize) + if (global.ServerSettings.sortingIgnorePrefix) { - bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`titleIgnorePrefix\`)`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) } else { - bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`title\`)`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitle})`), 'display_title']) } } @@ -716,13 +752,14 @@ module.exports = { bookWhere.push(...userPermissionBookWhere.bookWhere) let includeAttributes = [[Sequelize.literal('(SELECT max(mp.updatedAt) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id)'), 'recent_progress']] - let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = 0 OR mp.isFinished IS NULL)` + let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = ${booleanLiteral(false, Database.sequelize)} OR mp.isFinished IS NULL)` if (library.settings.onlyShowLaterBooksInContinueSeries) { - const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as FLOAT) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = 1 AND mp.userId = :userId AND bs.seriesId = series.id)` + const castType = Database.sequelize.getDialect() === 'postgres' ? 'DOUBLE PRECISION' : 'FLOAT' + const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as ${castType}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)` includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence']) - booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as FLOAT) > ${maxSequenceQuery}` + booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as ${castType}) > ${maxSequenceQuery}` } const { rows: series, count } = await Database.seriesModel.findAndCountAll({ @@ -735,7 +772,7 @@ module.exports = { }, // TODO: Simplify queries // Has at least 1 book finished - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = 1)`), { + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)})`), { [Sequelize.Op.gte]: 1 }), // Has at least 1 book not finished (that has a sequence number higher than the highest already read, if library config is toggled) @@ -743,7 +780,7 @@ module.exports = { [Sequelize.Op.gte]: 1 }), // Has no books in progress - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = 0 AND mp.currentTime > 0)`), 0) + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)} AND mp.currentTime > 0)`), 0) ], attributes: { include: includeAttributes @@ -760,7 +797,7 @@ module.exports = { order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], where: { '$book.mediaProgresses.isFinished$': { - [Sequelize.Op.or]: [null, 0] + [Sequelize.Op.or]: [null, false] } }, include: { @@ -854,7 +891,7 @@ module.exports = { { libraryId }, - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = 1 OR mp.currentTime > 0))`), 0) + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0))`), 0) ], replacements: { userId: user.id, @@ -904,6 +941,7 @@ module.exports = { id: { [Sequelize.Op.in]: booksFromSeriesToInclude } + } ] }, @@ -1130,7 +1168,7 @@ module.exports = { // Search narrators const narratorMatches = [] - const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, { + const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, ${jsonArrayExpand('b.narrators', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1147,7 +1185,7 @@ module.exports = { // Search tags const tagMatches = [] - const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.tags', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1164,7 +1202,7 @@ module.exports = { // Search genres const genreMatches = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1244,7 +1282,7 @@ module.exports = { */ async getGenresWithCount(libraryId) { const genres = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { replacements: { libraryId }, diff --git a/server/utils/queries/libraryItemsPodcastFilters.js b/server/utils/queries/libraryItemsPodcastFilters.js index 8bb5dc110..d89a4bd73 100644 --- a/server/utils/queries/libraryItemsPodcastFilters.js +++ b/server/utils/queries/libraryItemsPodcastFilters.js @@ -3,6 +3,7 @@ const Database = require('../../Database') const Logger = require('../../Logger') const { profile } = require('../../utils/profiler') const stringifySequelizeQuery = require('../stringifySequelizeQuery') +const { jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression, jsonArrayExpand, jsonPathNumber } = require('../sqlDialectHelpers') const countCache = new Map() @@ -24,10 +25,10 @@ module.exports = { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { replacements['userTagsSelected'] = user.permissions.itemTagsSelected if (user.permissions.selectedTagsNotAccessible) { - podcastWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0)) + podcastWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0)) } else { podcastWhere.push( - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), { + Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) ) @@ -53,7 +54,7 @@ module.exports = { const replacements = {} if (['genres', 'tags'].includes(group)) { - mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), { + mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) replacements.filterValue = value @@ -87,12 +88,12 @@ module.exports = { return [[Sequelize.literal('libraryItem.mtime'), dir]] } else if (sortBy === 'media.metadata.author') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' - return [[Sequelize.literal(`\`podcast\`.\`author\` COLLATE NOCASE ${nullDir}`)]] + return [[Sequelize.literal(`${noCaseSortExpression('podcast.author', Database.sequelize)} ${nullDir}`)]] } else if (sortBy === 'media.metadata.title') { if (global.ServerSettings.sortingIgnorePrefix) { - return [[Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]] } else { - return [[Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]] } } else if (sortBy === 'media.numTracks') { return [['numEpisodes', dir]] @@ -455,7 +456,7 @@ module.exports = { // Search tags const tagMatches = [] - const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.tags', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -472,7 +473,7 @@ module.exports = { // Search genres const genreMatches = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -563,12 +564,12 @@ module.exports = { * @returns {Promise<{ totalSize:number, totalDuration:number, numAudioFiles:number, totalItems:number}>} */ async getPodcastLibraryStats(libraryId) { - const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = "podcast" AND li.libraryId = :libraryId;`, { + const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = 'podcast' AND li.libraryId = :libraryId;`, { replacements: { libraryId } }) - const [statResults] = await Database.sequelize.query(`SELECT SUM(json_extract(pe.audioFile, '$.duration')) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, { + const [statResults] = await Database.sequelize.query(`SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, { replacements: { libraryId } @@ -588,7 +589,7 @@ module.exports = { */ async getGenresWithCount(libraryId) { const genres = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { replacements: { libraryId }, @@ -611,7 +612,7 @@ module.exports = { */ async getLongestPodcasts(libraryId, limit) { const podcasts = await Database.podcastModel.findAll({ - attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(json_extract(pe.audioFile, '$.duration')) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']], + attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']], include: { model: Database.libraryItemModel, attributes: ['id', 'libraryId'], diff --git a/server/utils/queries/seriesFilters.js b/server/utils/queries/seriesFilters.js index ed71e5b3f..3aa208552 100644 --- a/server/utils/queries/seriesFilters.js +++ b/server/utils/queries/seriesFilters.js @@ -2,6 +2,7 @@ const Sequelize = require('sequelize') const Logger = require('../../Logger') const Database = require('../../Database') const libraryItemsBookFilters = require('./libraryItemsBookFilters') +const { booleanLiteral, jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression } = require('../sqlDialectHelpers') module.exports = { decode(text) { @@ -60,7 +61,7 @@ module.exports = { // TODO: Simplify and break-out let attrQuery = null if (['genres', 'tags', 'narrators'].includes(filterGroup)) { - attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (SELECT count(*) FROM json_each(b.${filterGroup}) WHERE json_valid(b.${filterGroup}) AND json_each.value = :filterValue) > 0` + attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ${jsonArrayContainsValue(`b.${filterGroup}`, 'filterValue', Database.sequelize)} > 0` userPermissionBookWhere.replacements.filterValue = filterValue } else if (filterGroup === 'authors') { attrQuery = 'SELECT count(*) FROM books b, bookSeries bs, bookAuthors ba WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ba.bookId = b.id AND ba.authorId = :filterValue' @@ -73,18 +74,18 @@ module.exports = { userPermissionBookWhere.replacements.filterValue = filterValue } else if (filterGroup === 'progress') { if (filterValue === 'not-finished') { - attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)' + attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})` userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'finished') { - const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)' + const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})` seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0)) userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'not-started') { - const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = 1 OR mp.currentTime > 0)' + const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0)` seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0)) userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'in-progress') { - attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = 0' + attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)}` userPermissionBookWhere.replacements.userId = user.id } } @@ -95,13 +96,13 @@ module.exports = { if (!attrQuery) attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id' if (!user.canAccessExplicitContent) { - attrQuery += ' AND b.explicit = 0' + attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}` } if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (user.permissions.selectedTagsNotAccessible) { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0` } else { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0` } } } @@ -128,9 +129,9 @@ module.exports = { order.push(['createdAt', dir]) } else if (sortBy === 'name') { if (global.ServerSettings.sortingIgnorePrefix) { - order.push([Sequelize.literal('nameIgnorePrefix COLLATE NOCASE'), dir]) + order.push([Sequelize.literal(noCaseSortExpression('nameIgnorePrefix', Database.sequelize)), dir]) } else { - order.push([Sequelize.literal('`series`.`name` COLLATE NOCASE'), dir]) + order.push([Sequelize.literal(noCaseSortExpression('series.name', Database.sequelize)), dir]) } } else if (sortBy === 'totalDuration') { seriesAttributes.include.push([Sequelize.literal('(SELECT SUM(b.duration) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND b.id = bs.bookId)'), 'totalDuration']) diff --git a/server/utils/sqlDialectHelpers.js b/server/utils/sqlDialectHelpers.js new file mode 100644 index 000000000..36538bcb1 --- /dev/null +++ b/server/utils/sqlDialectHelpers.js @@ -0,0 +1,75 @@ +function getDialect(sequelize) { + if (!sequelize) return 'sqlite' + if (typeof sequelize.getDialect === 'function') return sequelize.getDialect() + return sequelize.dialect?.name || 'sqlite' +} + +function isPostgres(sequelize) { + return getDialect(sequelize) === 'postgres' +} + +function booleanLiteral(value, sequelize) { + if (isPostgres(sequelize)) return value ? 'TRUE' : 'FALSE' + return value ? '1' : '0' +} + +function noCaseSortExpression(columnExpression, sequelize) { + if (isPostgres(sequelize)) return `LOWER(${columnExpression})` + return `${columnExpression} COLLATE NOCASE` +} + +function coalesceFunctionName(sequelize) { + return isPostgres(sequelize) ? 'COALESCE' : 'IFNULL' +} + +function jsonArrayContainsAny(columnExpression, bindName, sequelize) { + if (isPostgres(sequelize)) { + return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:${bindName}))` + } + return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value IN (:${bindName}))` +} + +function jsonArrayContainsValue(columnExpression, bindName, sequelize) { + if (isPostgres(sequelize)) { + return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :${bindName})` + } + return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value = :${bindName})` +} + +function jsonArrayExpand(columnExpression, sequelize, options = {}) { + const alias = options.alias || 'json_each' + const textValues = options.textValues !== false + if (isPostgres(sequelize)) { + const fn = textValues ? 'jsonb_array_elements_text' : 'jsonb_array_elements' + return `${fn}(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS ${alias}(value)` + } + return `json_each(${columnExpression})` +} + +function jsonPathText(columnExpression, pathSegments, sequelize) { + const path = Array.isArray(pathSegments) ? pathSegments : [pathSegments] + if (isPostgres(sequelize)) { + return `${columnExpression}::jsonb #>> '{${path.join(',')}}'` + } + return `json_extract(${columnExpression}, '$.${path.join('.')}')` +} + +function jsonPathNumber(columnExpression, pathSegments, sequelize) { + if (isPostgres(sequelize)) { + return `NULLIF(${jsonPathText(columnExpression, pathSegments, sequelize)}, '')::double precision` + } + return `json_extract(${columnExpression}, '$.${[].concat(pathSegments).join('.')}')` +} + +module.exports = { + getDialect, + isPostgres, + booleanLiteral, + noCaseSortExpression, + coalesceFunctionName, + jsonArrayContainsAny, + jsonArrayContainsValue, + jsonArrayExpand, + jsonPathText, + jsonPathNumber +} diff --git a/test/server/utils/sqlDialectHelpers.test.js b/test/server/utils/sqlDialectHelpers.test.js new file mode 100644 index 000000000..006b02364 --- /dev/null +++ b/test/server/utils/sqlDialectHelpers.test.js @@ -0,0 +1,58 @@ +const { expect } = require('chai') + +const { + booleanLiteral, + noCaseSortExpression, + coalesceFunctionName, + jsonArrayContainsAny, + jsonArrayContainsValue, + jsonArrayExpand, + jsonPathText, + jsonPathNumber +} = require('../../../server/utils/sqlDialectHelpers') + +const sqlite = { + getDialect: () => 'sqlite' +} + +const postgres = { + getDialect: () => 'postgres' +} + +describe('sqlDialectHelpers', () => { + it('should return sqlite and postgres boolean literals', () => { + expect(booleanLiteral(true, sqlite)).to.equal('1') + expect(booleanLiteral(false, sqlite)).to.equal('0') + expect(booleanLiteral(true, postgres)).to.equal('TRUE') + expect(booleanLiteral(false, postgres)).to.equal('FALSE') + }) + + it('should generate case-insensitive sort expressions', () => { + expect(noCaseSortExpression('name', sqlite)).to.equal('name COLLATE NOCASE') + expect(noCaseSortExpression('name', postgres)).to.equal('LOWER(name)') + }) + + it('should choose the correct null-coalescing function name', () => { + expect(coalesceFunctionName(sqlite)).to.equal('IFNULL') + expect(coalesceFunctionName(postgres)).to.equal('COALESCE') + }) + + it('should generate array membership count queries per dialect', () => { + expect(jsonArrayContainsAny('tags', 'selectedTags', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:selectedTags))') + expect(jsonArrayContainsAny('tags', 'selectedTags', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:selectedTags))") + + expect(jsonArrayContainsValue('tags', 'tag', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value = :tag)') + expect(jsonArrayContainsValue('tags', 'tag', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :tag)") + }) + + it('should generate json array expansion and path extraction by dialect', () => { + expect(jsonArrayExpand('books.tags', sqlite)).to.equal('json_each(books.tags)') + expect(jsonArrayExpand('books.tags', postgres)).to.equal("jsonb_array_elements_text(COALESCE(books.tags::jsonb, '[]'::jsonb)) AS json_each(value)") + + expect(jsonPathText('payload', ['metadata', 'filename'], sqlite)).to.equal("json_extract(payload, '$.metadata.filename')") + expect(jsonPathText('payload', ['metadata', 'filename'], postgres)).to.equal("payload::jsonb #>> '{metadata,filename}'") + + expect(jsonPathNumber('payload', ['duration'], sqlite)).to.equal("json_extract(payload, '$.duration')") + expect(jsonPathNumber('payload', ['duration'], postgres)).to.equal("NULLIF(payload::jsonb #>> '{duration}', '')::double precision") + }) +}) From f31826c256e0f01e3e0b1a10f79b79c43bbdc847 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 16:58:07 -0500 Subject: [PATCH 02/19] fix migration metadata table detection on postgres --- server/managers/MigrationManager.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index 588771318..df0778849 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -215,7 +215,7 @@ class MigrationManager { async checkOrCreateMigrationsMetaTable() { const queryInterface = this.sequelize.getQueryInterface() - let migrationsMetaTableExists = await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE) + let migrationsMetaTableExists = await this.tableExists(MigrationManager.MIGRATIONS_META_TABLE) // If the table exists, check that the `version` and `maxVersion` rows exist if (migrationsMetaTableExists) { @@ -255,6 +255,20 @@ class MigrationManager { } } + async tableExists(tableName) { + const queryInterface = this.sequelize.getQueryInterface() + if (typeof queryInterface.tableExists === 'function') { + return queryInterface.tableExists(tableName) + } + + const tables = await queryInterface.showAllTables() + return tables.some((table) => { + if (typeof table === 'string') return table === tableName + if (table?.tableName) return table.tableName === tableName + return false + }) + } + extractVersionFromTag(tag) { if (!tag) return null const versionMatch = tag.match(/^v?(\d+\.\d+\.\d+)/) From 920a7434f1f8d36fff3f79fd985da7f79641b51e Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:03:03 -0500 Subject: [PATCH 03/19] handle postgres alias casing in migration metadata --- server/managers/MigrationManager.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index df0778849..51ded7c89 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -202,15 +202,15 @@ class MigrationManager { const migrationsMetaTable = `"${MigrationManager.MIGRATIONS_META_TABLE}"` await this.checkOrCreateMigrationsMetaTable() - const [{ version }] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, { + const [versionRow] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, { type: Sequelize.QueryTypes.SELECT }) - this.databaseVersion = version + this.databaseVersion = versionRow?.version - const [{ maxVersion }] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, { + const [maxVersionRow] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, { type: Sequelize.QueryTypes.SELECT }) - this.maxVersion = maxVersion + this.maxVersion = maxVersionRow?.maxVersion || maxVersionRow?.maxversion } async checkOrCreateMigrationsMetaTable() { From c7d14f7d303bf4bc355c923409b61a33c79df6cf Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:07:10 -0500 Subject: [PATCH 04/19] disable identifier quoting for postgres compatibility --- server/Database.js | 3 ++- server/managers/MigrationManager.js | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/server/Database.js b/server/Database.js index 68f6e2729..3d48980c1 100644 --- a/server/Database.js +++ b/server/Database.js @@ -296,7 +296,8 @@ class Database { this.sequelize = new Sequelize(process.env.DATABASE_URL, { dialect: 'postgres', logging: logging, - benchmark: benchmark + benchmark: benchmark, + quoteIdentifiers: false }) } diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index 51ded7c89..99c89e155 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -199,7 +199,7 @@ class MigrationManager { } async fetchVersionsFromDatabase() { - const migrationsMetaTable = `"${MigrationManager.MIGRATIONS_META_TABLE}"` + const migrationsMetaTable = MigrationManager.MIGRATIONS_META_TABLE await this.checkOrCreateMigrationsMetaTable() const [versionRow] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, { @@ -219,7 +219,7 @@ class MigrationManager { // If the table exists, check that the `version` and `maxVersion` rows exist if (migrationsMetaTableExists) { - const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM "${MigrationManager.MIGRATIONS_META_TABLE}" WHERE key IN ('version', 'maxVersion')`, { + const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM ${MigrationManager.MIGRATIONS_META_TABLE} WHERE key IN ('version', 'maxVersion')`, { type: Sequelize.QueryTypes.SELECT }) if (count < 2) { @@ -247,7 +247,7 @@ class MigrationManager { allowNull: false } }) - await this.sequelize.query(`INSERT INTO "${MigrationManager.MIGRATIONS_META_TABLE}" (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, { + await this.sequelize.query(`INSERT INTO ${MigrationManager.MIGRATIONS_META_TABLE} (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, { replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0' }, type: Sequelize.QueryTypes.INSERT }) @@ -319,7 +319,7 @@ class MigrationManager { async updateMaxVersion() { try { - await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :maxVersion WHERE key = 'maxVersion'`, { + await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :maxVersion WHERE key = 'maxVersion'`, { replacements: { maxVersion: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) @@ -331,7 +331,7 @@ class MigrationManager { async updateDatabaseVersion() { try { - await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :version WHERE key = 'version'`, { + await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :version WHERE key = 'version'`, { replacements: { version: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) From c0a4ce139690ccfc0d2afc4a7c30162a3f79466b Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:12:12 -0500 Subject: [PATCH 05/19] make sqlite to postgres migrator case-insensitive --- server/scripts/migrateSqliteToPostgres.js | 70 +++++++++++++++-------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js index 60acc11e2..f5c49a8ef 100644 --- a/server/scripts/migrateSqliteToPostgres.js +++ b/server/scripts/migrateSqliteToPostgres.js @@ -132,13 +132,24 @@ async function main() { `SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`, [PG_SCHEMA] ) - const pgTables = new Set(pgTablesRows.rows.map((row) => row.table_name)) - const tablesToMigrate = sqliteTables.filter((table) => pgTables.has(table)) + const pgTablesByLowerName = new Map(pgTablesRows.rows.map((row) => [row.table_name.toLowerCase(), row.table_name])) + + const tablesToMigrate = sqliteTables + .map((sqliteTable) => { + const postgresTable = pgTablesByLowerName.get(sqliteTable.toLowerCase()) + if (!postgresTable) return null + return { + sqliteTable, + postgresTable + } + }) + .filter(Boolean) + tablesToMigrate.sort((a, b) => { - const ai = preferredOrder.indexOf(a) - const bi = preferredOrder.indexOf(b) - if (ai === -1 && bi === -1) return a.localeCompare(b) + const ai = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === a.sqliteTable.toLowerCase()) + const bi = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === b.sqliteTable.toLowerCase()) + if (ai === -1 && bi === -1) return a.sqliteTable.localeCompare(b.sqliteTable) if (ai === -1) return 1 if (bi === -1) return -1 return ai - bi @@ -148,34 +159,46 @@ async function main() { throw new Error('No overlapping tables found between SQLite and PostgreSQL') } - console.log(`[migrate] tables to migrate: ${tablesToMigrate.join(', ')}`) + console.log(`[migrate] tables to migrate: ${tablesToMigrate.map((table) => table.sqliteTable).join(', ')}`) const pgColumnsByTable = new Map() - for (const table of tablesToMigrate) { + for (const { postgresTable } of tablesToMigrate) { const columnsResult = await pg.query( `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, - [PG_SCHEMA, table] + [PG_SCHEMA, postgresTable] ) - const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name, column])) - pgColumnsByTable.set(table, columnMap) + const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column])) + pgColumnsByTable.set(postgresTable, columnMap) } if (!DRY_RUN) { await pg.query('BEGIN') await pg.query('SET session_replication_role = replica') - const truncateList = tablesToMigrate.map((table) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`).join(', ') + const truncateList = tablesToMigrate.map(({ postgresTable }) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`).join(', ') await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`) console.log('[migrate] truncated target tables') } - for (const table of tablesToMigrate) { - const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(table)}`) - const pgColumns = pgColumnsByTable.get(table) - const insertColumns = rows.length ? Object.keys(rows[0]).filter((column) => pgColumns.has(column)) : [] + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(sqliteTable)}`) + const pgColumns = pgColumnsByTable.get(postgresTable) + const insertColumns = rows.length + ? Object.keys(rows[0]) + .map((sqliteColumn) => { + const pgColumn = pgColumns.get(sqliteColumn.toLowerCase()) + if (!pgColumn) return null + return { + sqliteColumn, + postgresColumn: pgColumn.column_name, + metadata: pgColumn + } + }) + .filter(Boolean) + : [] if (!rows.length || !insertColumns.length) { - console.log(`[migrate] ${table}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`) + console.log(`[migrate] ${sqliteTable}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`) continue } @@ -189,19 +212,18 @@ async function main() { for (const row of batchRows) { const placeholders = [] for (const column of insertColumns) { - const pgColumn = pgColumns.get(column) - params.push(convertValue(row[column], pgColumn)) + params.push(convertValue(row[column.sqliteColumn], column.metadata)) placeholders.push(`$${paramIndex++}`) } valuesSql.push(`(${placeholders.join(', ')})`) } - const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)} (${insertColumns.map(quoteIdent).join(', ')}) VALUES ${valuesSql.join(', ')}` + const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}` await pg.query(insertSql, params) } } - console.log(`[migrate] ${table}: ${rows.length} rows`) + console.log(`[migrate] ${sqliteTable}: ${rows.length} rows`) } if (!DRY_RUN) { @@ -211,11 +233,11 @@ async function main() { } const parity = [] - for (const table of tablesToMigrate) { - const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(table)}`) - const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`) + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(sqliteTable)}`) + const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`) parity.push({ - table, + table: sqliteTable, sqliteCount: Number(sqliteCountRow.count || 0), postgresCount: Number(pgCountResult.rows[0].count || 0) }) From 8593721c3558e3f0e2f998c264cb250be8fd2394 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:15:45 -0500 Subject: [PATCH 06/19] harden json normalization in postgres migration script --- server/scripts/migrateSqliteToPostgres.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js index f5c49a8ef..e8b72881c 100644 --- a/server/scripts/migrateSqliteToPostgres.js +++ b/server/scripts/migrateSqliteToPostgres.js @@ -89,10 +89,22 @@ function normalizeBoolean(value) { function normalizeJson(value) { if (value === null || value === undefined || value === '') return null if (typeof value === 'object') return value + + const textValue = String(value) + try { - return JSON.parse(value) + const parsed = JSON.parse(textValue) + if (typeof parsed === 'string') { + try { + return JSON.parse(parsed) + } catch (error) { + return parsed + } + } + return parsed } catch (error) { - return value + // Keep non-JSON payloads as JSON string values so inserts remain valid. + return JSON.stringify(textValue) } } From cb31fd34d56723ba09573b30753bd9d0ed14e8ab Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:45:32 -0500 Subject: [PATCH 07/19] prevent lossy postgres migration edge cases --- server/Database.js | 5 + server/models/Book.js | 2 +- server/models/PlaybackSession.js | 2 +- server/scripts/migrateSqliteToPostgres.js | 255 ++++++++++++++++-- .../scripts/migrateSqliteToPostgres.test.js | 131 +++++++++ 5 files changed, 372 insertions(+), 23 deletions(-) create mode 100644 test/server/scripts/migrateSqliteToPostgres.test.js diff --git a/server/Database.js b/server/Database.js index 3d48980c1..3cbf32819 100644 --- a/server/Database.js +++ b/server/Database.js @@ -412,6 +412,11 @@ class Database { require('./models/CustomMetadataProvider').init(this.sequelize) require('./models/MediaItemShare').init(this.sequelize) + if (this.isPostgresDialect() && !force && !this.isNew) { + Logger.info('[Database] Skipping sequelize.sync for existing postgres schema') + return Promise.resolve() + } + return this.sequelize.sync({ force, alter: false }) } diff --git a/server/models/Book.js b/server/models/Book.js index 08982bb03..ac71d207e 100644 --- a/server/models/Book.js +++ b/server/models/Book.js @@ -148,7 +148,7 @@ class Book extends Model { }, title: DataTypes.STRING, titleIgnorePrefix: DataTypes.STRING, - subtitle: DataTypes.STRING, + subtitle: DataTypes.TEXT, publishedYear: DataTypes.STRING, publishedDate: DataTypes.STRING, publisher: DataTypes.STRING, diff --git a/server/models/PlaybackSession.js b/server/models/PlaybackSession.js index 196fbda6c..47526d5f6 100644 --- a/server/models/PlaybackSession.js +++ b/server/models/PlaybackSession.js @@ -190,7 +190,7 @@ class PlaybackSession extends Model { currentTime: DataTypes.FLOAT, serverVersion: DataTypes.STRING, coverPath: DataTypes.STRING, - timeListening: DataTypes.INTEGER, + timeListening: DataTypes.FLOAT, mediaMetadata: DataTypes.JSON, date: DataTypes.STRING, dayOfWeek: DataTypes.STRING, diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js index e8b72881c..43dc439da 100644 --- a/server/scripts/migrateSqliteToPostgres.js +++ b/server/scripts/migrateSqliteToPostgres.js @@ -9,11 +9,6 @@ const PG_SCHEMA = process.env.PG_SCHEMA || 'public' const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500) const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true' -if (!DATABASE_URL) { - console.error('[migrate] DATABASE_URL is required') - process.exit(1) -} - const preferredOrder = [ 'migrationsMeta', 'SequelizeMeta', @@ -75,6 +70,120 @@ function sqliteGet(db, sql, params = []) { }) } +async function findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) { + const issues = [] + + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const pgColumns = pgColumnsByTable.get(postgresTable) + if (!pgColumns) continue + + for (const column of pgColumns.values()) { + if (column.data_type !== 'character varying' || !column.character_maximum_length) continue + + const sqliteColumn = column.column_name + const maxLength = Number(column.character_maximum_length) + const quotedTable = quoteIdent(sqliteTable) + const quotedColumn = quoteIdent(sqliteColumn) + + try { + const maxLengthRow = await sqliteGet( + sqliteDb, + `SELECT MAX(LENGTH(${quotedColumn})) AS maxLength FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL` + ) + const actualMaxLength = Number(maxLengthRow?.maxLength || 0) + if (actualMaxLength <= maxLength) continue + + const overCountRow = await sqliteGet( + sqliteDb, + `SELECT COUNT(*) AS count FROM ${quotedTable} WHERE LENGTH(${quotedColumn}) > ?`, + [maxLength] + ) + + issues.push({ + sqliteTable, + sqliteColumn, + postgresTable, + postgresColumn: column.column_name, + maxLength, + actualMaxLength, + overCount: Number(overCountRow?.count || 0) + }) + } catch (error) { + // Ignore columns missing in sqlite source table + } + } + } + + return issues +} + +function isIntegerCompatible(value) { + if (value === null || value === undefined) return true + + if (typeof value === 'number') { + return Number.isFinite(value) && Number.isInteger(value) + } + + if (typeof value === 'string') { + const trimmed = value.trim() + if (!trimmed) return false + if (!/^-?\d+$/.test(trimmed)) return false + const parsed = Number(trimmed) + return Number.isFinite(parsed) + } + + return false +} + +async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) { + const issues = [] + + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const pgColumns = pgColumnsByTable.get(postgresTable) + if (!pgColumns) continue + + for (const column of pgColumns.values()) { + const dataType = column.data_type + if (dataType !== 'smallint' && dataType !== 'integer' && dataType !== 'bigint') continue + + const sqliteColumn = column.column_name + const quotedTable = quoteIdent(sqliteTable) + const quotedColumn = quoteIdent(sqliteColumn) + + let rows + try { + rows = await sqliteAll(sqliteDb, `SELECT ${quotedColumn} AS value FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`) + } catch (error) { + // Ignore columns missing in sqlite source table + continue + } + + let badCount = 0 + let sampleValue = null + for (const row of rows) { + if (!isIntegerCompatible(row.value)) { + badCount += 1 + if (sampleValue === null) sampleValue = row.value + } + } + + if (badCount > 0) { + issues.push({ + sqliteTable, + sqliteColumn, + postgresTable, + postgresColumn: column.column_name, + postgresType: dataType, + badCount, + sampleValue + }) + } + } + } + + return issues +} + function normalizeBoolean(value) { if (value === null || value === undefined) return null if (typeof value === 'boolean') return value @@ -88,7 +197,7 @@ function normalizeBoolean(value) { function normalizeJson(value) { if (value === null || value === undefined || value === '') return null - if (typeof value === 'object') return value + if (typeof value === 'object') return JSON.stringify(value) const textValue = String(value) @@ -96,14 +205,14 @@ function normalizeJson(value) { const parsed = JSON.parse(textValue) if (typeof parsed === 'string') { try { - return JSON.parse(parsed) + return JSON.stringify(JSON.parse(parsed)) } catch (error) { - return parsed + return JSON.stringify(parsed) } } - return parsed + return JSON.stringify(parsed) } catch (error) { - // Keep non-JSON payloads as JSON string values so inserts remain valid. + // Keep non-JSON payloads as JSON string values so inserts remain valid JSON. return JSON.stringify(textValue) } } @@ -121,10 +230,45 @@ function convertValue(value, pgColumn) { return normalizeJson(value) } + if ((dataType === 'smallint' || dataType === 'integer' || dataType === 'bigint') && value !== null && value !== undefined) { + if (typeof value === 'number') return value + if (typeof value === 'string' && /^-?\d+$/.test(value.trim())) { + return Number(value) + } + } + return value } +function getOverlongColumns(row, insertColumns) { + const overlong = [] + + for (const column of insertColumns) { + const maxLength = column.metadata.character_maximum_length + if (!maxLength) continue + + const value = row[column.sqliteColumn] + if (value === null || value === undefined) continue + + const length = String(value).length + if (length > maxLength) { + overlong.push({ + sqliteColumn: column.sqliteColumn, + postgresColumn: column.postgresColumn, + maxLength, + actualLength: length + }) + } + } + + return overlong +} + async function main() { + if (!DATABASE_URL) { + throw new Error('DATABASE_URL is required') + } + console.log(`[migrate] sqlite source: ${SQLITE_PATH}`) console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`) console.log(`[migrate] dry run: ${DRY_RUN}`) @@ -176,13 +320,38 @@ async function main() { const pgColumnsByTable = new Map() for (const { postgresTable } of tablesToMigrate) { const columnsResult = await pg.query( - `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, + `SELECT column_name, data_type, udt_name, character_maximum_length FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, [PG_SCHEMA, postgresTable] ) const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column])) pgColumnsByTable.set(postgresTable, columnMap) } + const overlongVarcharIssues = await findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) + const integerTypeIssues = await findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) + + if (overlongVarcharIssues.length) { + console.error('[migrate] overlong source values detected for varchar columns:') + overlongVarcharIssues.forEach((issue) => { + console.error( + `[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` + + `(max=${issue.maxLength}, actualMax=${issue.actualMaxLength}, overRows=${issue.overCount})` + ) + }) + throw new Error('Migration aborted to prevent truncation/data loss. Widen target column types first.') + } + + if (integerTypeIssues.length) { + console.error('[migrate] non-integer source values detected for integer columns:') + integerTypeIssues.forEach((issue) => { + console.error( + `[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` + + `(type=${issue.postgresType}, badRows=${issue.badCount}, sample=${JSON.stringify(issue.sampleValue)})` + ) + }) + throw new Error('Migration aborted to prevent numeric precision loss. Widen target numeric column types first.') + } + if (!DRY_RUN) { await pg.query('BEGIN') await pg.query('SET session_replication_role = replica') @@ -222,16 +391,49 @@ async function main() { let paramIndex = 1 for (const row of batchRows) { - const placeholders = [] - for (const column of insertColumns) { - params.push(convertValue(row[column.sqliteColumn], column.metadata)) - placeholders.push(`$${paramIndex++}`) - } + const placeholders = [] + for (const column of insertColumns) { + params.push(convertValue(row[column.sqliteColumn], column.metadata)) + placeholders.push(`$${paramIndex++}`) + } valuesSql.push(`(${placeholders.join(', ')})`) } const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}` - await pg.query(insertSql, params) + + try { + await pg.query('SAVEPOINT migrate_batch') + await pg.query(insertSql, params) + await pg.query('RELEASE SAVEPOINT migrate_batch') + } catch (error) { + await pg.query('ROLLBACK TO SAVEPOINT migrate_batch') + console.error(`[migrate] batch insert failed for ${sqliteTable} (offset=${offset}, size=${batchRows.length}): ${error.message}`) + + for (let rowIndex = 0; rowIndex < batchRows.length; rowIndex++) { + const row = batchRows[rowIndex] + const singleRowParams = insertColumns.map((column) => convertValue(row[column.sqliteColumn], column.metadata)) + const singleRowInsertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES (${singleRowParams.map((_, index) => `$${index + 1}`).join(', ')})` + + try { + await pg.query('SAVEPOINT migrate_row') + await pg.query(singleRowInsertSql, singleRowParams) + await pg.query('RELEASE SAVEPOINT migrate_row') + } catch (rowError) { + await pg.query('ROLLBACK TO SAVEPOINT migrate_row') + const overlongColumns = getOverlongColumns(row, insertColumns) + if (overlongColumns.length) { + overlongColumns.forEach((column) => { + console.error( + `[migrate] overlong value in ${sqliteTable}.${column.sqliteColumn} -> ${postgresTable}.${column.postgresColumn} ` + + `(length=${column.actualLength}, max=${column.maxLength})` + ) + }) + } + + throw rowError + } + } + } } } @@ -271,7 +473,18 @@ async function main() { } } -main().catch((error) => { - console.error('[migrate] failed:', error) - process.exit(1) -}) +if (require.main === module) { + main().catch((error) => { + console.error('[migrate] failed:', error) + process.exit(1) + }) +} + +module.exports = { + normalizeJson, + isIntegerCompatible, + convertValue, + findOverlongVarcharValues, + findIntegerTypeIssues, + quoteIdent +} diff --git a/test/server/scripts/migrateSqliteToPostgres.test.js b/test/server/scripts/migrateSqliteToPostgres.test.js new file mode 100644 index 000000000..9b218079e --- /dev/null +++ b/test/server/scripts/migrateSqliteToPostgres.test.js @@ -0,0 +1,131 @@ +const { expect } = require('chai') +const sqlite3 = require('sqlite3') + +const { + normalizeJson, + isIntegerCompatible, + findOverlongVarcharValues, + findIntegerTypeIssues +} = require('../../../server/scripts/migrateSqliteToPostgres') + +function openMemoryDb() { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(':memory:', (error) => { + if (error) return reject(error) + resolve(db) + }) + }) +} + +function run(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.run(sql, params, (error) => { + if (error) return reject(error) + resolve() + }) + }) +} + +function close(db) { + return new Promise((resolve, reject) => { + db.close((error) => { + if (error) return reject(error) + resolve() + }) + }) +} + +describe('migrateSqliteToPostgres script helpers', () => { + it('should keep malformed JSON payloads insertable by returning valid JSON text', () => { + const malformed = '"{"x",1}"' + const normalized = normalizeJson(malformed) + + expect(() => JSON.parse(normalized)).to.not.throw() + expect(JSON.parse(normalized)).to.equal('"{"x",1}"') + }) + + it('should detect overlong varchar values before migration', async () => { + const db = await openMemoryDb() + + try { + await run(db, 'CREATE TABLE books (subtitle TEXT)') + await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['x'.repeat(300)]) + await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['ok']) + + const tablesToMigrate = [{ sqliteTable: 'books', postgresTable: 'books' }] + const pgColumnsByTable = new Map([ + [ + 'books', + new Map([ + [ + 'subtitle', + { + column_name: 'subtitle', + data_type: 'character varying', + character_maximum_length: 255 + } + ] + ]) + ] + ]) + + const issues = await findOverlongVarcharValues(db, tablesToMigrate, pgColumnsByTable) + + expect(issues).to.have.length(1) + expect(issues[0]).to.include({ + sqliteTable: 'books', + sqliteColumn: 'subtitle', + postgresTable: 'books', + postgresColumn: 'subtitle', + maxLength: 255, + actualMaxLength: 300, + overCount: 1 + }) + } finally { + await close(db) + } + }) + + it('should detect non-integer values for integer target columns', async () => { + const db = await openMemoryDb() + + try { + await run(db, 'CREATE TABLE playbackSessions (timeListening REAL)') + await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [25.802536999999997]) + await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [42]) + + const tablesToMigrate = [{ sqliteTable: 'playbackSessions', postgresTable: 'playbackSessions' }] + const pgColumnsByTable = new Map([ + [ + 'playbackSessions', + new Map([ + [ + 'timelistening', + { + column_name: 'timeListening', + data_type: 'integer' + } + ] + ]) + ] + ]) + + const issues = await findIntegerTypeIssues(db, tablesToMigrate, pgColumnsByTable) + + expect(issues).to.have.length(1) + expect(issues[0].sqliteTable).to.equal('playbackSessions') + expect(issues[0].sqliteColumn).to.equal('timeListening') + expect(issues[0].postgresType).to.equal('integer') + expect(issues[0].badCount).to.equal(1) + } finally { + await close(db) + } + }) + + it('should only accept integer-compatible values for integer columns', () => { + expect(isIntegerCompatible(10)).to.equal(true) + expect(isIntegerCompatible('10')).to.equal(true) + expect(isIntegerCompatible(10.5)).to.equal(false) + expect(isIntegerCompatible('10.5')).to.equal(false) + }) +}) From 6227a60fed3a09340b21596ee8864044e5414dcd Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 17:50:08 -0500 Subject: [PATCH 08/19] fix postgres legacy user id lookup --- server/models/User.js | 4 +++- test/server/models/User.test.js | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 test/server/models/User.test.js diff --git a/server/models/User.js b/server/models/User.js index 936efde13..c7ee78dbd 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -422,9 +422,11 @@ class User extends Model { const cachedUser = userCache.getById(userId) || userCache.getByOldId(userId) if (cachedUser) return cachedUser + const idMatcher = this.sequelize.getDialect() === 'postgres' ? sequelize.where(sequelize.cast(sequelize.col('user.id'), 'text'), userId) : { id: userId } + const user = await this.findOne({ where: { - [sequelize.Op.or]: [{ id: userId }, { 'extraData.oldUserId': userId }] + [sequelize.Op.or]: [idMatcher, { 'extraData.oldUserId': userId }] }, include: this.sequelize.models.mediaProgress }) diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js new file mode 100644 index 000000000..9e2bf5888 --- /dev/null +++ b/test/server/models/User.test.js @@ -0,0 +1,42 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const sequelize = require('sequelize') + +const User = require('../../../server/models/User') + +describe('User model', () => { + describe('getUserByIdOrOldId', () => { + let originalSequelize + + beforeEach(() => { + originalSequelize = User.sequelize + }) + + afterEach(() => { + User.sequelize = originalSequelize + sinon.restore() + }) + + it('should cast id to text on postgres when checking legacy token ids', async () => { + User.sequelize = { + getDialect: () => 'postgres', + models: { + mediaProgress: {} + } + } + + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + await User.getUserByIdOrOldId('root') + + expect(findOneStub.calledOnce).to.equal(true) + + const options = findOneStub.firstCall.args[0] + const whereOr = options.where[sequelize.Op.or] + + expect(whereOr).to.have.length(2) + expect(whereOr[0].attribute.type).to.equal('text') + expect(whereOr[1]).to.deep.equal({ 'extraData.oldUserId': 'root' }) + }) + }) +}) From 15d2715a7408e888c1cd1b5f781b06ea49b4f5da Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 18:02:36 -0500 Subject: [PATCH 09/19] fix postgres old user id json lookup --- server/models/User.js | 20 ++++++++++--- test/server/models/User.test.js | 50 +++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/server/models/User.js b/server/models/User.js index c7ee78dbd..c260ad08e 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -422,12 +422,24 @@ class User extends Model { const cachedUser = userCache.getById(userId) || userCache.getByOldId(userId) if (cachedUser) return cachedUser - const idMatcher = this.sequelize.getDialect() === 'postgres' ? sequelize.where(sequelize.cast(sequelize.col('user.id'), 'text'), userId) : { id: userId } + const isUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userId) + if (isUuid) { + const byId = await this.findByPk(userId, { + include: this.sequelize.models.mediaProgress + }) + if (byId) { + userCache.set(byId) + return byId + } + } + + const oldIdMatcher = + this.sequelize.getDialect() === 'postgres' + ? sequelize.where(sequelize.literal(`"extraData"#>>'{oldUserId}'`), userId) + : { 'extraData.oldUserId': userId } const user = await this.findOne({ - where: { - [sequelize.Op.or]: [idMatcher, { 'extraData.oldUserId': userId }] - }, + where: oldIdMatcher, include: this.sequelize.models.mediaProgress }) diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js index 9e2bf5888..ce187f93e 100644 --- a/test/server/models/User.test.js +++ b/test/server/models/User.test.js @@ -1,7 +1,5 @@ const { expect } = require('chai') const sinon = require('sinon') -const sequelize = require('sequelize') - const User = require('../../../server/models/User') describe('User model', () => { @@ -17,7 +15,7 @@ describe('User model', () => { sinon.restore() }) - it('should cast id to text on postgres when checking legacy token ids', async () => { + it('should resolve UUID ids via primary-key lookup on postgres', async () => { User.sequelize = { getDialect: () => 'postgres', models: { @@ -25,6 +23,46 @@ describe('User model', () => { } } + const user = { id: 'e8e677b2-da16-4220-ab67-443b7714caf9' } + const findByPkStub = sinon.stub(User, 'findByPk').resolves(user) + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + const result = await User.getUserByIdOrOldId('e8e677b2-da16-4220-ab67-443b7714caf9') + + expect(result).to.equal(user) + expect(findByPkStub.calledOnce).to.equal(true) + expect(findOneStub.called).to.equal(false) + }) + + it('should query legacy oldUserId with postgres-safe JSON matcher', async () => { + User.sequelize = { + getDialect: () => 'postgres', + models: { + mediaProgress: {} + } + } + + const findByPkStub = sinon.stub(User, 'findByPk').resolves(null) + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + await User.getUserByIdOrOldId('root') + + expect(findByPkStub.called).to.equal(false) + expect(findOneStub.calledOnce).to.equal(true) + + const options = findOneStub.firstCall.args[0] + expect(options.where.attribute.val).to.equal('"extraData"#>>\'{oldUserId}\'') + expect(options.where.logic).to.equal('root') + }) + + it('should keep sqlite oldUserId matcher unchanged', async () => { + User.sequelize = { + getDialect: () => 'sqlite', + models: { + mediaProgress: {} + } + } + const findOneStub = sinon.stub(User, 'findOne').resolves(null) await User.getUserByIdOrOldId('root') @@ -32,11 +70,7 @@ describe('User model', () => { expect(findOneStub.calledOnce).to.equal(true) const options = findOneStub.firstCall.args[0] - const whereOr = options.where[sequelize.Op.or] - - expect(whereOr).to.have.length(2) - expect(whereOr[0].attribute.type).to.equal('text') - expect(whereOr[1]).to.deep.equal({ 'extraData.oldUserId': 'root' }) + expect(options.where).to.deep.equal({ 'extraData.oldUserId': 'root' }) }) }) }) From 8b3f67b7c942c8f3bd8bcbc7e03a52fb0ed9dd8a Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 18:20:58 -0500 Subject: [PATCH 10/19] guard numeric casts for postgres text fields --- server/models/Series.js | 3 +- .../utils/queries/libraryItemsBookFilters.js | 34 +++++++------------ server/utils/sqlDialectHelpers.js | 18 +++++++++- test/server/utils/sqlDialectHelpers.test.js | 12 ++++++- 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/server/models/Series.js b/server/models/Series.js index 28450a010..192378b7d 100644 --- a/server/models/Series.js +++ b/server/models/Series.js @@ -1,6 +1,7 @@ const { DataTypes, Model, where, fn, col, literal } = require('sequelize') const { getTitlePrefixAtEnd, getTitleIgnorePrefix } = require('../utils/index') +const { safeTextToDoubleExpression } = require('../utils/sqlDialectHelpers') class Series extends Model { constructor(values, options) { @@ -158,7 +159,7 @@ class Series extends Model { } } ], - order: [[literal('CAST(`bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]] + order: [[literal(`${safeTextToDoubleExpression(this.sequelize.getDialect() === 'postgres' ? '"bookSeries"."sequence"' : '`bookSeries.sequence`', this.sequelize)} ASC NULLS LAST`)]] }) } diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 029448529..993946c71 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -6,7 +6,7 @@ const authorFilters = require('./authorFilters') const ShareManager = require('../../managers/ShareManager') const { profile } = require('../profiler') const stringifySequelizeQuery = require('../stringifySequelizeQuery') -const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand } = require('../sqlDialectHelpers') +const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand, safeTextToDoubleExpression, safeTextToIntegerExpression } = require('../sqlDialectHelpers') const countCache = new Map() module.exports = { @@ -237,7 +237,7 @@ module.exports = { } else if (group === 'publishedDecades') { const startYear = parseInt(value) const endYear = parseInt(value, 10) + 9 - mediaWhere = Sequelize.where(Sequelize.literal('CAST(publishedYear AS INTEGER)'), { + mediaWhere = Sequelize.where(Sequelize.literal(safeTextToIntegerExpression('publishedYear', Database.sequelize)), { [Sequelize.Op.between]: [startYear, endYear] }) } @@ -274,7 +274,7 @@ module.exports = { } else if (sortBy === 'media.duration') { return [['duration', dir]] } else if (sortBy === 'media.metadata.publishedYear') { - return [[Sequelize.literal('CAST(book.publishedYear AS INTEGER)'), dir]] + return [[Sequelize.literal(safeTextToIntegerExpression('book.publishedYear', Database.sequelize)), dir]] } else if (sortBy === 'media.metadata.authorNameLF') { // Sort by author name last first, secondary sort by title return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesLastFirst', Database.sequelize)), dir], getTitleOrder()] @@ -288,10 +288,8 @@ module.exports = { return [getTitleOrder()] } else if (sortBy === 'sequence') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' - if (Database.sequelize.getDialect() === 'postgres') { - return [[Sequelize.literal(`CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ${nullDir}`)]] - } - return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]] + const sequenceColumn = Database.sequelize.getDialect() === 'postgres' ? '"series.bookSeries"."sequence"' : '`series.bookSeries.sequence`' + return [[Sequelize.literal(`${safeTextToDoubleExpression(sequenceColumn, Database.sequelize)} ${nullDir}`)]] } else if (sortBy === 'progress') { return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]] } else if (sortBy === 'progress.createdAt') { @@ -331,11 +329,7 @@ module.exports = { } ], order: [ - Sequelize.literal( - Database.sequelize.getDialect() === 'postgres' - ? 'CAST("books.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST' - : 'CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST' - ) + Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"books.bookSeries"."sequence"' : '`books.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) ] }) const bookSeriesToInclude = [] @@ -529,11 +523,7 @@ module.exports = { if (sortBy !== 'sequence') { // Secondary sort by sequence sortOrder.push([ - Sequelize.literal( - Database.sequelize.getDialect() === 'postgres' - ? 'CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST' - : 'CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST' - ) + Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"series.bookSeries"."sequence"' : '`series.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) ]) } } else if (filterGroup === 'issues') { @@ -755,11 +745,11 @@ module.exports = { let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = ${booleanLiteral(false, Database.sequelize)} OR mp.isFinished IS NULL)` if (library.settings.onlyShowLaterBooksInContinueSeries) { - const castType = Database.sequelize.getDialect() === 'postgres' ? 'DOUBLE PRECISION' : 'FLOAT' - const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as ${castType}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)` + const safeSequenceExpr = safeTextToDoubleExpression('bs.sequence', Database.sequelize) + const maxSequenceQuery = `(SELECT max(${safeSequenceExpr}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)` includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence']) - booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as ${castType}) > ${maxSequenceQuery}` + booksNotFinishedQuery = booksNotFinishedQuery + ` AND ${safeSequenceExpr} > ${maxSequenceQuery}` } const { rows: series, count } = await Database.seriesModel.findAndCountAll({ @@ -794,7 +784,7 @@ module.exports = { attributes: ['bookId', 'sequence'], separate: true, subQuery: false, - order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], + order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]], where: { '$book.mediaProgresses.isFinished$': { [Sequelize.Op.or]: [null, false] @@ -907,7 +897,7 @@ module.exports = { model: Database.bookModel, where: userPermissionBookWhere.bookWhere }, - order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], + order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]], limit: 1 }, subQuery: false, diff --git a/server/utils/sqlDialectHelpers.js b/server/utils/sqlDialectHelpers.js index 36538bcb1..7f6f93118 100644 --- a/server/utils/sqlDialectHelpers.js +++ b/server/utils/sqlDialectHelpers.js @@ -61,6 +61,20 @@ function jsonPathNumber(columnExpression, pathSegments, sequelize) { return `json_extract(${columnExpression}, '$.${[].concat(pathSegments).join('.')}')` } +function safeTextToDoubleExpression(columnExpression, sequelize) { + if (isPostgres(sequelize)) { + return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(${columnExpression})::double precision ELSE NULL END` + } + return `CAST(${columnExpression} AS FLOAT)` +} + +function safeTextToIntegerExpression(columnExpression, sequelize) { + if (isPostgres(sequelize)) { + return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?\\d+$' THEN BTRIM(${columnExpression})::integer ELSE NULL END` + } + return `CAST(${columnExpression} AS INTEGER)` +} + module.exports = { getDialect, isPostgres, @@ -71,5 +85,7 @@ module.exports = { jsonArrayContainsValue, jsonArrayExpand, jsonPathText, - jsonPathNumber + jsonPathNumber, + safeTextToDoubleExpression, + safeTextToIntegerExpression } diff --git a/test/server/utils/sqlDialectHelpers.test.js b/test/server/utils/sqlDialectHelpers.test.js index 006b02364..82d9154d8 100644 --- a/test/server/utils/sqlDialectHelpers.test.js +++ b/test/server/utils/sqlDialectHelpers.test.js @@ -8,7 +8,9 @@ const { jsonArrayContainsValue, jsonArrayExpand, jsonPathText, - jsonPathNumber + jsonPathNumber, + safeTextToDoubleExpression, + safeTextToIntegerExpression } = require('../../../server/utils/sqlDialectHelpers') const sqlite = { @@ -55,4 +57,12 @@ describe('sqlDialectHelpers', () => { expect(jsonPathNumber('payload', ['duration'], sqlite)).to.equal("json_extract(payload, '$.duration')") expect(jsonPathNumber('payload', ['duration'], postgres)).to.equal("NULLIF(payload::jsonb #>> '{duration}', '')::double precision") }) + + it('should generate safe numeric cast expressions for sequence-like text', () => { + expect(safeTextToDoubleExpression('sequence', sqlite)).to.equal('CAST(sequence AS FLOAT)') + expect(safeTextToDoubleExpression('"bookSeries"."sequence"', postgres)).to.equal("CASE WHEN BTRIM(\"bookSeries\".\"sequence\") ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(\"bookSeries\".\"sequence\")::double precision ELSE NULL END") + + expect(safeTextToIntegerExpression('publishedYear', sqlite)).to.equal('CAST(publishedYear AS INTEGER)') + expect(safeTextToIntegerExpression('book.publishedYear', postgres)).to.equal("CASE WHEN BTRIM(book.publishedYear) ~ '^[+-]?\\d+$' THEN BTRIM(book.publishedYear)::integer ELSE NULL END") + }) }) From 29e49950893f67c4f59ba2eacfd87e3e89b22857 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 19:33:47 -0500 Subject: [PATCH 11/19] fix postgres series join alias in sequence sort --- server/utils/queries/libraryItemsBookFilters.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 993946c71..1933a0fbc 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -288,7 +288,7 @@ module.exports = { return [getTitleOrder()] } else if (sortBy === 'sequence') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' - const sequenceColumn = Database.sequelize.getDialect() === 'postgres' ? '"series.bookSeries"."sequence"' : '`series.bookSeries.sequence`' + const sequenceColumn = Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`' return [[Sequelize.literal(`${safeTextToDoubleExpression(sequenceColumn, Database.sequelize)} ${nullDir}`)]] } else if (sortBy === 'progress') { return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]] @@ -329,7 +329,7 @@ module.exports = { } ], order: [ - Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"books.bookSeries"."sequence"' : '`books.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) + Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"books->bookSeries"."sequence"' : '`books.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) ] }) const bookSeriesToInclude = [] @@ -523,7 +523,7 @@ module.exports = { if (sortBy !== 'sequence') { // Secondary sort by sequence sortOrder.push([ - Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"series.bookSeries"."sequence"' : '`series.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) + Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`) ]) } } else if (filterGroup === 'issues') { From 7f7e73e3afebe6621239c6dc93119a3fff75a801 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 19:46:21 -0500 Subject: [PATCH 12/19] fix postgres collapse-series id quoting --- server/utils/queries/libraryItemsBookFilters.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 1933a0fbc..d896e24a0 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -619,11 +619,12 @@ module.exports = { const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."title"' : '`libraryItem`.`title`' const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."titleIgnorePrefix"' : '`libraryItem`.`titleIgnorePrefix`' const fallbackFn = coalesceFunctionName(Database.sequelize) + const includedBookSeriesIds = bookSeriesToInclude.map((v) => Database.sequelize.escape(v.id)).join(', ') if (global.ServerSettings.sortingIgnorePrefix) { - bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds})), ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) } else { - bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitle})`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds})), ${fallbackLibraryItemTitle})`), 'display_title']) } } From d5b509a9d7c489c673a777bf79744fe2c7be1790 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 19:51:07 -0500 Subject: [PATCH 13/19] fix postgres media progress timestamp update --- server/models/MediaProgress.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/models/MediaProgress.js b/server/models/MediaProgress.js index 0ebe2f598..88b759660 100644 --- a/server/models/MediaProgress.js +++ b/server/models/MediaProgress.js @@ -253,7 +253,13 @@ class MediaProgress extends Model { const escapedDate = this.sequelize.escape(new Date(progressPayload.lastUpdate)) Logger.info(`[MediaProgress] Manually setting updatedAt to ${escapedDate} (media item ${this.mediaItemId})`) - await this.sequelize.query(`UPDATE "mediaProgresses" SET "updatedAt" = ${escapedDate} WHERE "id" = '${this.id}'`) + await this.constructor.update( + { updatedAt: new Date(progressPayload.lastUpdate) }, + { + where: { id: this.id }, + silent: true + } + ) await this.reload() } From 372a9292550ed33263102808656de9cdd76747e1 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 19:55:23 -0500 Subject: [PATCH 14/19] fix postgres oldUserId column casing --- server/models/User.js | 2 +- test/server/models/User.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/models/User.js b/server/models/User.js index c260ad08e..7afdd7da3 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -435,7 +435,7 @@ class User extends Model { const oldIdMatcher = this.sequelize.getDialect() === 'postgres' - ? sequelize.where(sequelize.literal(`"extraData"#>>'{oldUserId}'`), userId) + ? sequelize.where(sequelize.literal(`extradata#>>'{oldUserId}'`), userId) : { 'extraData.oldUserId': userId } const user = await this.findOne({ diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js index ce187f93e..82c289c45 100644 --- a/test/server/models/User.test.js +++ b/test/server/models/User.test.js @@ -51,7 +51,7 @@ describe('User model', () => { expect(findOneStub.calledOnce).to.equal(true) const options = findOneStub.firstCall.args[0] - expect(options.where.attribute.val).to.equal('"extraData"#>>\'{oldUserId}\'') + expect(options.where.attribute.val).to.equal("extradata#>>'{oldUserId}'") expect(options.where.logic).to.equal('root') }) From 51895f67321dfcacd11759e33dc8bc04b9efd44b Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 20:15:03 -0500 Subject: [PATCH 15/19] fix empty collapse-series list query and add diagnostics --- .../utils/queries/libraryItemsBookFilters.js | 61 +++++++++++++------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index d896e24a0..e3392c03d 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -362,26 +362,32 @@ module.exports = { async findAndCountAll(findOptions, limit, offset, useCountCache) { const model = Database.bookModel - if (useCountCache) { - const countCacheKey = stringifySequelizeQuery(findOptions) - Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`) - if (!countCache.has(countCacheKey)) { - const count = await model.count(findOptions) - countCache.set(countCacheKey, count) - } - - findOptions.limit = limit || null - findOptions.offset = offset - - const rows = await model.findAll(findOptions) - - return { rows, count: countCache.get(countCacheKey) } - } - findOptions.limit = limit || null findOptions.offset = offset - return await model.findAndCountAll(findOptions) + try { + if (useCountCache) { + const countCacheKey = stringifySequelizeQuery(findOptions) + Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`) + if (!countCache.has(countCacheKey)) { + const count = await model.count(findOptions) + countCache.set(countCacheKey, count) + } + + const rows = await model.findAll(findOptions) + return { rows, count: countCache.get(countCacheKey) } + } + + return await model.findAndCountAll(findOptions) + } catch (error) { + Logger.error(`[LibraryItemsBookFilters] findAndCountAll failed: ${error.message}`) + try { + Logger.error(`[LibraryItemsBookFilters] findAndCountAll query: ${stringifySequelizeQuery(findOptions)}`) + } catch (stringifyError) { + Logger.error(`[LibraryItemsBookFilters] failed to stringify query: ${stringifyError.message}`) + } + throw error + } }, /** @@ -620,11 +626,28 @@ module.exports = { const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."titleIgnorePrefix"' : '`libraryItem`.`titleIgnorePrefix`' const fallbackFn = coalesceFunctionName(Database.sequelize) const includedBookSeriesIds = bookSeriesToInclude.map((v) => Database.sequelize.escape(v.id)).join(', ') + const collapseSeriesSubqueryByName = includedBookSeriesIds + ? `(SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))` + : 'NULL' + const collapseSeriesSubqueryByNameIgnorePrefix = includedBookSeriesIds + ? `(SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))` + : 'NULL' + + Logger.debug( + `[LibraryItemsBookFilters] collapse-series computed includeIds=${bookSeriesToInclude.length} excludeBooks=${booksToExclude.length} ` + + `filterGroup=${filterGroup || 'none'} filterValue=${filterValue || 'none'} sortBy=${sortBy}` + ) + if (!includedBookSeriesIds) { + Logger.warn( + `[LibraryItemsBookFilters] collapse-series produced no include IDs; using library item title fallback ` + + `(libraryId=${libraryId}, filterGroup=${filterGroup || 'none'}, filterValue=${filterValue || 'none'}, sortBy=${sortBy})` + ) + } if (global.ServerSettings.sortingIgnorePrefix) { - bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds})), ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByNameIgnorePrefix}, ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) } else { - bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds})), ${fallbackLibraryItemTitle})`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByName}, ${fallbackLibraryItemTitle})`), 'display_title']) } } From 13e333a1bf135db8e2987b3edb8581c00547feae Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 20:32:12 -0500 Subject: [PATCH 16/19] fix postgres collapsed-series display title alias --- server/utils/queries/libraryItemsBookFilters.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index e3392c03d..62e2c55d9 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -622,8 +622,8 @@ module.exports = { // When collapsing series and sorting by title then use the series name instead of the book title // for this set an attribute "display_title" to use in sorting - const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."title"' : '`libraryItem`.`title`' - const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."titleIgnorePrefix"' : '`libraryItem`.`titleIgnorePrefix`' + const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.title' : '`libraryItem`.`title`' + const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.titleIgnorePrefix' : '`libraryItem`.`titleIgnorePrefix`' const fallbackFn = coalesceFunctionName(Database.sequelize) const includedBookSeriesIds = bookSeriesToInclude.map((v) => Database.sequelize.escape(v.id)).join(', ') const collapseSeriesSubqueryByName = includedBookSeriesIds From 06ad6a722f7af930d6cf2940dc51b71519dd1228 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Mon, 2 Mar 2026 22:08:15 -0500 Subject: [PATCH 17/19] add regression tests for postgres query edge cases --- test/server/models/MediaProgress.test.js | 70 +++++++++ .../scripts/migrateSqliteToPostgres.test.js | 15 ++ .../queries/libraryItemsBookFilters.test.js | 133 ++++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 test/server/models/MediaProgress.test.js create mode 100644 test/server/utils/queries/libraryItemsBookFilters.test.js diff --git a/test/server/models/MediaProgress.test.js b/test/server/models/MediaProgress.test.js new file mode 100644 index 000000000..a7c490306 --- /dev/null +++ b/test/server/models/MediaProgress.test.js @@ -0,0 +1,70 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Logger = require('../../../server/Logger') +const MediaProgress = require('../../../server/models/MediaProgress') + +function createProgressInstance() { + const progress = Object.create(MediaProgress.prototype) + + Object.defineProperties(progress, { + id: { value: 'progress-1', writable: true, configurable: true }, + mediaItemId: { value: 'media-1', writable: true, configurable: true }, + duration: { value: 3600, writable: true, configurable: true }, + currentTime: { value: 120, writable: true, configurable: true }, + isFinished: { value: false, writable: true, configurable: true }, + hideFromContinueListening: { value: false, writable: true, configurable: true }, + extraData: { value: {}, writable: true, configurable: true } + }) + + progress.changed = sinon.stub().returns(false) + progress.set = sinon.stub().callsFake((payload) => Object.assign(progress, payload)) + progress.save = sinon.stub().resolves() + progress.reload = sinon.stub().resolves() + progress.constructor = { + update: sinon.stub().resolves(), + sequelize: { + escape: (value) => `'${value.toISOString ? value.toISOString() : value}'` + } + } + + return progress +} + +describe('MediaProgress', () => { + afterEach(() => { + sinon.restore() + }) + + describe('applyProgressUpdate', () => { + it('should update updatedAt via model update for valid lastUpdate', async () => { + const progress = createProgressInstance() + const infoStub = sinon.stub(Logger, 'info') + const lastUpdate = '2026-03-03T01:00:00.000Z' + + await progress.applyProgressUpdate({ currentTime: 130, lastUpdate }) + + expect(progress.save.calledOnce).to.equal(true) + expect(progress.constructor.update.calledOnce).to.equal(true) + expect(progress.constructor.update.firstCall.args[0].updatedAt.toISOString()).to.equal(new Date(lastUpdate).toISOString()) + expect(progress.constructor.update.firstCall.args[1]).to.deep.equal({ + where: { id: 'progress-1' }, + silent: true + }) + expect(progress.reload.calledOnce).to.equal(true) + expect(infoStub.calledWithMatch('[MediaProgress] Manually setting updatedAt')).to.equal(true) + }) + + it('should skip manual updatedAt update when lastUpdate is invalid', async () => { + const progress = createProgressInstance() + const warnStub = sinon.stub(Logger, 'warn') + + await progress.applyProgressUpdate({ currentTime: 130, lastUpdate: 'invalid-date' }) + + expect(progress.save.calledOnce).to.equal(true) + expect(progress.constructor.update.called).to.equal(false) + expect(progress.reload.called).to.equal(false) + expect(warnStub.calledWithMatch('[MediaProgress] Invalid date provided for lastUpdate')).to.equal(true) + }) + }) +}) diff --git a/test/server/scripts/migrateSqliteToPostgres.test.js b/test/server/scripts/migrateSqliteToPostgres.test.js index 9b218079e..d0bb3d592 100644 --- a/test/server/scripts/migrateSqliteToPostgres.test.js +++ b/test/server/scripts/migrateSqliteToPostgres.test.js @@ -4,6 +4,7 @@ const sqlite3 = require('sqlite3') const { normalizeJson, isIntegerCompatible, + convertValue, findOverlongVarcharValues, findIntegerTypeIssues } = require('../../../server/scripts/migrateSqliteToPostgres') @@ -128,4 +129,18 @@ describe('migrateSqliteToPostgres script helpers', () => { expect(isIntegerCompatible(10.5)).to.equal(false) expect(isIntegerCompatible('10.5')).to.equal(false) }) + + it('should coerce integer-like strings for postgres integer columns', () => { + expect(convertValue('42', { data_type: 'integer' })).to.equal(42) + expect(convertValue('-7', { data_type: 'bigint' })).to.equal(-7) + expect(convertValue('4.2', { data_type: 'integer' })).to.equal('4.2') + }) + + it('should always return valid json text for postgres json columns', () => { + const convertedObject = convertValue({ a: 1 }, { data_type: 'jsonb', udt_name: 'jsonb' }) + const convertedMalformed = convertValue('"{"x",1}"', { data_type: 'jsonb', udt_name: 'jsonb' }) + + expect(JSON.parse(convertedObject)).to.deep.equal({ a: 1 }) + expect(JSON.parse(convertedMalformed)).to.equal('"{"x",1}"') + }) }) diff --git a/test/server/utils/queries/libraryItemsBookFilters.test.js b/test/server/utils/queries/libraryItemsBookFilters.test.js new file mode 100644 index 000000000..b014c1399 --- /dev/null +++ b/test/server/utils/queries/libraryItemsBookFilters.test.js @@ -0,0 +1,133 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Database = require('../../../../server/Database') +const Logger = require('../../../../server/Logger') +const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters') + +function createSequelizeStub(dialect, models) { + return { + getDialect: () => dialect, + escape: (value) => `'${String(value).replace(/'/g, "''")}'`, + random: () => ({ fn: 'random' }), + models + } +} + +function createModelStubs() { + return { + book: { + findAndCountAll: sinon.stub(), + findAll: sinon.stub(), + count: sinon.stub() + }, + libraryItem: {}, + feed: {}, + bookSeries: {}, + series: { + findAll: sinon.stub() + }, + bookAuthor: {}, + author: {}, + mediaProgress: {}, + mediaItemShare: {} + } +} + +describe('libraryItemsBookFilters postgres query safety', () => { + let originalSequelize + let originalServerSettings + let modelStubs + + beforeEach(() => { + originalSequelize = Database.sequelize + originalServerSettings = global.ServerSettings + modelStubs = createModelStubs() + global.ServerSettings = { + sortingIgnorePrefix: false + } + }) + + afterEach(() => { + Database.sequelize = originalSequelize + global.ServerSettings = originalServerSettings + sinon.restore() + }) + + it('should avoid generating IN () when collapse-series has no include ids', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 }) + + sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({ + booksToExclude: [], + bookSeriesToInclude: [] + }) + const warnStub = sinon.stub(Logger, 'warn') + + await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, true, [], 20, 0) + + const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0] + const displayTitleExpression = findOptions.attributes.include[0][0].val + + expect(displayTitleExpression).to.include('COALESCE(NULL, libraryItem.title)') + expect(displayTitleExpression).to.not.include('IN ()') + expect(warnStub.calledOnce).to.equal(true) + }) + + it('should escape collapse-series ids safely for postgres subquery', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 }) + + sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({ + booksToExclude: [], + bookSeriesToInclude: [{ id: "series-id-'quoted'", numBooks: 2, libraryItemIds: [] }] + }) + global.ServerSettings.sortingIgnorePrefix = true + + await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.title', false, true, [], 20, 0) + + const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0] + const displayTitleExpression = findOptions.attributes.include[0][0].val + + expect(displayTitleExpression).to.include("bs.id IN ('series-id-''quoted''')") + expect(displayTitleExpression).to.include('libraryItem.titleIgnorePrefix') + }) + + it('should use postgres-safe join alias for sequence sorting', () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + + const order = libraryItemsBookFilters.getOrder('sequence', false, false) + const expression = order[0][0].val + + expect(expression).to.include('"series->bookSeries"."sequence"') + expect(expression).to.include('CASE WHEN BTRIM') + }) + + it('should log generated query when findAndCountAll fails', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.book.findAndCountAll.rejects(new Error('boom')) + const errorStub = sinon.stub(Logger, 'error') + + try { + await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, false, [], 20, 0) + expect.fail('Expected getFilteredLibraryItems to throw') + } catch (error) { + expect(error.message).to.equal('boom') + } + + expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll failed: boom')).to.equal(true) + expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll query:')).to.equal(true) + }) + + it('should use postgres-safe books->bookSeries alias in collapse-series selector query', async () => { + Database.sequelize = createSequelizeStub('postgres', modelStubs) + modelStubs.series.findAll.resolves([]) + + await libraryItemsBookFilters.getCollapseSeriesBooksToExclude({ where: {}, include: [] }, null) + + const findAllOptions = modelStubs.series.findAll.firstCall.args[0] + const orderExpression = findAllOptions.order[0].val + + expect(orderExpression).to.include('"books->bookSeries"."sequence"') + }) +}) From 7d97d091917d28a8acd4f3fd98e9d11ccaac2c0b Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Tue, 3 Mar 2026 15:12:38 -0500 Subject: [PATCH 18/19] tighten postgres compatibility tests and diagnostics --- .../utils/queries/libraryItemsBookFilters.js | 2 +- test/server/Database.test.js | 101 +++++++++++++++++ test/server/managers/MigrationManager.test.js | 60 ++++++++++ test/server/models/User.test.js | 20 ++++ .../queries/libraryItemsBookFilters.test.js | 4 +- .../libraryItemsPodcastFilters.test.js | 75 ++++++++++++ .../utils/queries/seriesFilters.test.js | 107 ++++++++++++++++++ 7 files changed, 366 insertions(+), 3 deletions(-) create mode 100644 test/server/Database.test.js create mode 100644 test/server/utils/queries/libraryItemsPodcastFilters.test.js create mode 100644 test/server/utils/queries/seriesFilters.test.js diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 62e2c55d9..35e0f9bb3 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -638,7 +638,7 @@ module.exports = { `filterGroup=${filterGroup || 'none'} filterValue=${filterValue || 'none'} sortBy=${sortBy}` ) if (!includedBookSeriesIds) { - Logger.warn( + Logger.debug( `[LibraryItemsBookFilters] collapse-series produced no include IDs; using library item title fallback ` + `(libraryId=${libraryId}, filterGroup=${filterGroup || 'none'}, filterValue=${filterValue || 'none'}, sortBy=${sortBy})` ) diff --git a/test/server/Database.test.js b/test/server/Database.test.js new file mode 100644 index 000000000..a70e1d204 --- /dev/null +++ b/test/server/Database.test.js @@ -0,0 +1,101 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const fs = require('../../server/libs/fsExtra') +const Database = require('../../server/Database') + +describe('Database', () => { + let originalDialect + let originalSequelize + let originalEnv + + beforeEach(() => { + originalDialect = Database.dialect + originalSequelize = Database.sequelize + originalEnv = { + DB_DIALECT: process.env.DB_DIALECT, + DATABASE_URL: process.env.DATABASE_URL + } + }) + + afterEach(() => { + Database.dialect = originalDialect + Database.sequelize = originalSequelize + + if (originalEnv.DB_DIALECT === undefined) delete process.env.DB_DIALECT + else process.env.DB_DIALECT = originalEnv.DB_DIALECT + + if (originalEnv.DATABASE_URL === undefined) delete process.env.DATABASE_URL + else process.env.DATABASE_URL = originalEnv.DATABASE_URL + + sinon.restore() + }) + + describe('getConfiguredDialect', () => { + it('should default to sqlite when no env variables are set', () => { + delete process.env.DB_DIALECT + delete process.env.DATABASE_URL + + expect(Database.getConfiguredDialect()).to.equal('sqlite') + }) + + it('should use explicit DB_DIALECT value when valid', () => { + process.env.DB_DIALECT = 'postgres' + process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite' + + expect(Database.getConfiguredDialect()).to.equal('postgres') + }) + + it('should infer postgres dialect from DATABASE_URL', () => { + delete process.env.DB_DIALECT + process.env.DATABASE_URL = 'postgresql://user:pass@localhost:5432/abs' + + expect(Database.getConfiguredDialect()).to.equal('postgres') + }) + + it('should fallback to sqlite for unsupported DB_DIALECT', () => { + process.env.DB_DIALECT = 'mysql' + process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite' + + expect(Database.getConfiguredDialect()).to.equal('sqlite') + }) + }) + + describe('checkHasDb', () => { + it('should not check sqlite file existence in postgres mode', async () => { + Database.dialect = 'postgres' + const pathExistsStub = sinon.stub(fs, 'pathExists') + + const hasDb = await Database.checkHasDb() + + expect(hasDb).to.equal(true) + expect(pathExistsStub.called).to.equal(false) + }) + }) + + describe('checkHasTables', () => { + it('should return true when at least one table exists', async () => { + Database.sequelize = { + getQueryInterface: () => ({ + showAllTables: sinon.stub().resolves(['users']) + }) + } + + const hasTables = await Database.checkHasTables() + + expect(hasTables).to.equal(true) + }) + + it('should return false when no tables exist', async () => { + Database.sequelize = { + getQueryInterface: () => ({ + showAllTables: sinon.stub().resolves([]) + }) + } + + const hasTables = await Database.checkHasTables() + + expect(hasTables).to.equal(false) + }) + }) +}) diff --git a/test/server/managers/MigrationManager.test.js b/test/server/managers/MigrationManager.test.js index af2e9da8f..a306eb5be 100644 --- a/test/server/managers/MigrationManager.test.js +++ b/test/server/managers/MigrationManager.test.js @@ -189,6 +189,52 @@ describe('MigrationManager', () => { expect(loggerInfoStub.calledWith(sinon.match('Restored the original database'))).to.be.true expect(processExitStub.calledOnce).to.be.true }) + + it('should skip sqlite backup workflow for postgres migrations', async () => { + // Arrange + migrationManager.serverVersion = '1.2.0' + migrationManager.databaseVersion = '1.1.0' + migrationManager.maxVersion = '1.1.0' + migrationManager.initialized = true + sequelizeStub.getDialect.returns('postgres') + + umzugStub.migrations.resolves([{ name: 'v1.2.0-migration.js' }]) + umzugStub.executed.resolves([{ name: 'v1.1.0-migration.js' }]) + + // Act + await migrationManager.runMigrations() + + // Assert + expect(umzugStub.up.calledOnce).to.be.true + expect(fsCopyStub.called).to.be.false + expect(fsRemoveStub.called).to.be.false + }) + }) + + describe('tableExists', () => { + it('should use queryInterface.tableExists when available', async () => { + const tableExistsStub = sinon.stub().resolves(true) + const sequelize = sinon.createStubInstance(Sequelize) + sequelize.getQueryInterface.returns({ tableExists: tableExistsStub }) + const manager = new MigrationManager(sequelize, false, configPath) + + const exists = await manager.tableExists('migrationsMeta') + + expect(exists).to.equal(true) + expect(tableExistsStub.calledOnceWithExactly('migrationsMeta')).to.equal(true) + }) + + it('should fallback to showAllTables and support object table names', async () => { + const sequelize = sinon.createStubInstance(Sequelize) + sequelize.getQueryInterface.returns({ + showAllTables: sinon.stub().resolves([{ tableName: 'migrationsMeta' }]) + }) + const manager = new MigrationManager(sequelize, false, configPath) + + const exists = await manager.tableExists('migrationsMeta') + + expect(exists).to.equal(true) + }) }) describe('fetchVersionsFromDatabase', () => { @@ -280,6 +326,20 @@ describe('MigrationManager', () => { expect(error.message).to.equal('Database query failed') } }) + + it('should support lowercase maxversion alias from postgres drivers', async () => { + const sequelize = sinon.createStubInstance(Sequelize) + sequelize.query.onFirstCall().resolves([{ version: '1.1.0' }]) + sequelize.query.onSecondCall().resolves([{ maxversion: '1.2.0' }]) + + const manager = new MigrationManager(sequelize, false, configPath) + manager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves() + + await manager.fetchVersionsFromDatabase() + + expect(manager.databaseVersion).to.equal('1.1.0') + expect(manager.maxVersion).to.equal('1.2.0') + }) }) describe('updateMaxVersion', () => { diff --git a/test/server/models/User.test.js b/test/server/models/User.test.js index 82c289c45..5b935fee6 100644 --- a/test/server/models/User.test.js +++ b/test/server/models/User.test.js @@ -34,6 +34,26 @@ describe('User model', () => { expect(findOneStub.called).to.equal(false) }) + it('should resolve uppercase UUID ids via primary-key lookup on postgres', async () => { + User.sequelize = { + getDialect: () => 'postgres', + models: { + mediaProgress: {} + } + } + + const uppercaseUuid = 'E8E677B2-DA16-4220-AB67-443B7714CAF9' + const user = { id: uppercaseUuid } + const findByPkStub = sinon.stub(User, 'findByPk').resolves(user) + const findOneStub = sinon.stub(User, 'findOne').resolves(null) + + const result = await User.getUserByIdOrOldId(uppercaseUuid) + + expect(result).to.equal(user) + expect(findByPkStub.calledOnceWithExactly(uppercaseUuid, { include: User.sequelize.models.mediaProgress })).to.equal(true) + expect(findOneStub.called).to.equal(false) + }) + it('should query legacy oldUserId with postgres-safe JSON matcher', async () => { User.sequelize = { getDialect: () => 'postgres', diff --git a/test/server/utils/queries/libraryItemsBookFilters.test.js b/test/server/utils/queries/libraryItemsBookFilters.test.js index b014c1399..19b6a58bc 100644 --- a/test/server/utils/queries/libraryItemsBookFilters.test.js +++ b/test/server/utils/queries/libraryItemsBookFilters.test.js @@ -62,7 +62,7 @@ describe('libraryItemsBookFilters postgres query safety', () => { booksToExclude: [], bookSeriesToInclude: [] }) - const warnStub = sinon.stub(Logger, 'warn') + const debugStub = sinon.stub(Logger, 'debug') await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, true, [], 20, 0) @@ -71,7 +71,7 @@ describe('libraryItemsBookFilters postgres query safety', () => { expect(displayTitleExpression).to.include('COALESCE(NULL, libraryItem.title)') expect(displayTitleExpression).to.not.include('IN ()') - expect(warnStub.calledOnce).to.equal(true) + expect(debugStub.calledWithMatch('collapse-series produced no include IDs')).to.equal(true) }) it('should escape collapse-series ids safely for postgres subquery', async () => { diff --git a/test/server/utils/queries/libraryItemsPodcastFilters.test.js b/test/server/utils/queries/libraryItemsPodcastFilters.test.js new file mode 100644 index 000000000..79938abdc --- /dev/null +++ b/test/server/utils/queries/libraryItemsPodcastFilters.test.js @@ -0,0 +1,75 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Database = require('../../../../server/Database') +const libraryItemsPodcastFilters = require('../../../../server/utils/queries/libraryItemsPodcastFilters') + +describe('libraryItemsPodcastFilters dialect behavior', () => { + let originalSequelize + let originalServerSettings + + beforeEach(() => { + originalSequelize = Database.sequelize + originalServerSettings = global.ServerSettings + global.ServerSettings = { sortingIgnorePrefix: false } + }) + + afterEach(() => { + Database.sequelize = originalSequelize + global.ServerSettings = originalServerSettings + sinon.restore() + }) + + it('should build postgres json tag permission predicate', () => { + Database.sequelize = { getDialect: () => 'postgres' } + + const { podcastWhere, replacements } = libraryItemsPodcastFilters.getUserPermissionPodcastWhereQuery({ + canAccessExplicitContent: true, + permissions: { + accessAllTags: false, + itemTagsSelected: ['fiction'], + selectedTagsNotAccessible: false + } + }) + + expect(replacements.userTagsSelected).to.deep.equal(['fiction']) + expect(podcastWhere[0].attribute.val).to.include('jsonb_array_elements_text') + }) + + it('should build sqlite no-case author sort expression', () => { + Database.sequelize = { getDialect: () => 'sqlite' } + + const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false) + + expect(order[0][0].val).to.include('podcast.author COLLATE NOCASE') + }) + + it('should build postgres lower author sort expression', () => { + Database.sequelize = { getDialect: () => 'postgres' } + + const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false) + + expect(order[0][0].val).to.include('LOWER(podcast.author)') + }) + + it('should use postgres json duration extraction in podcast stats query', async () => { + const queryStub = sinon.stub() + queryStub.onFirstCall().resolves([[{ totalSize: 1000 }]]) + queryStub.onSecondCall().resolves([[{ totalDuration: '12.3', totalItems: '4', numAudioFiles: '9' }]]) + + Database.sequelize = { + getDialect: () => 'postgres', + query: queryStub + } + + const result = await libraryItemsPodcastFilters.getPodcastLibraryStats('library-1') + + expect(queryStub.secondCall.args[0]).to.include('NULLIF(pe.audioFile::jsonb #>>') + expect(result).to.deep.equal({ + totalSize: 1000, + totalDuration: '12.3', + numAudioFiles: '9', + totalItems: '4' + }) + }) +}) diff --git a/test/server/utils/queries/seriesFilters.test.js b/test/server/utils/queries/seriesFilters.test.js new file mode 100644 index 000000000..eb1c3cb54 --- /dev/null +++ b/test/server/utils/queries/seriesFilters.test.js @@ -0,0 +1,107 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const Database = require('../../../../server/Database') +const seriesFilters = require('../../../../server/utils/queries/seriesFilters') +const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters') + +function encodedFilter(group, value) { + return `${group}.${encodeURIComponent(Buffer.from(value).toString('base64'))}` +} + +describe('seriesFilters dialect behavior', () => { + let originalSequelize + let originalServerSettings + let modelStubs + + function setDialect(dialect) { + Database.sequelize = { + getDialect: () => dialect, + random: sinon.stub(), + models: modelStubs + } + } + + beforeEach(() => { + originalSequelize = Database.sequelize + originalServerSettings = global.ServerSettings + modelStubs = { + series: { + findAndCountAll: sinon.stub().resolves({ rows: [], count: 0 }) + }, + bookSeries: {}, + book: {}, + libraryItem: {}, + author: {}, + feed: {} + } + + global.ServerSettings = { sortingIgnorePrefix: false } + sinon.stub(libraryItemsBookFilters, 'getUserPermissionBookWhereQuery').returns({ + bookWhere: [], + replacements: {} + }) + }) + + afterEach(() => { + Database.sequelize = originalSequelize + global.ServerSettings = originalServerSettings + sinon.restore() + }) + + it('should build postgres progress filter with TRUE and FALSE literals', async () => { + setDialect('postgres') + + await seriesFilters.getFilteredSeries( + { id: 'library-1', settings: { hideSingleBookSeries: false } }, + { id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } }, + encodedFilter('progress', 'not-started'), + 'name', + false, + [], + 10, + 0 + ) + + const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0] + const progressWhere = findOptions.where[1] + + expect(progressWhere.attribute.val).to.include('mp.isFinished = TRUE') + }) + + it('should build sqlite no-case name sort expression', async () => { + setDialect('sqlite') + + await seriesFilters.getFilteredSeries( + { id: 'library-1', settings: { hideSingleBookSeries: false } }, + { id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } }, + null, + 'name', + false, + [], + 10, + 0 + ) + + const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0] + expect(findOptions.order[0][0].val).to.equal('series.name COLLATE NOCASE') + }) + + it('should build postgres lower-case name sort expression', async () => { + setDialect('postgres') + + await seriesFilters.getFilteredSeries( + { id: 'library-1', settings: { hideSingleBookSeries: false } }, + { id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } }, + null, + 'name', + false, + [], + 10, + 0 + ) + + const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0] + expect(findOptions.order[0][0].val).to.equal('LOWER(series.name)') + }) +}) From 024f892902570afef93a165e515d702b18b7c763 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Tue, 3 Mar 2026 17:17:01 -0500 Subject: [PATCH 19/19] skip sqlite-only migrations on postgres --- .../v2.15.0-series-column-unique.js | 8 +++- server/migrations/v2.15.1-reindex-nocase.js | 6 +++ server/migrations/v2.17.3-fk-constraints.js | 6 +++ .../v2.19.1-copy-title-to-library-items.js | 12 ++++++ .../v2.19.4-improve-podcast-queries.js | 12 ++++++ .../v2.15.0-series-column-unique.test.js | 17 ++++++++ .../migrations/v2.15.1-reindex-nocase.test.js | 41 +++++++++++++++++++ .../migrations/v2.17.3-fk-constraints.test.js | 10 +++++ ...2.19.1-copy-title-to-library-items.test.js | 12 ++++++ .../v2.19.4-improve-podcast-queries.test.js | 12 ++++++ 10 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 test/server/migrations/v2.15.1-reindex-nocase.test.js diff --git a/server/migrations/v2.15.0-series-column-unique.js b/server/migrations/v2.15.0-series-column-unique.js index 7f8526f97..be782225a 100644 --- a/server/migrations/v2.15.0-series-column-unique.js +++ b/server/migrations/v2.15.0-series-column-unique.js @@ -19,8 +19,12 @@ async function up({ context: { queryInterface, logger } }) { logger.info('[2.15.0 migration] UPGRADE BEGIN: 2.15.0-series-column-unique ') // Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0 - logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') - await queryInterface.sequelize.query('REINDEX NOCASE;') + if (queryInterface.sequelize.getDialect() === 'sqlite') { + logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') + await queryInterface.sequelize.query('REINDEX NOCASE;') + } else { + logger.info('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect') + } // Check if the unique index already exists const seriesIndexes = await queryInterface.showIndex('Series') diff --git a/server/migrations/v2.15.1-reindex-nocase.js b/server/migrations/v2.15.1-reindex-nocase.js index 2ec9487d2..429edf9ff 100644 --- a/server/migrations/v2.15.1-reindex-nocase.js +++ b/server/migrations/v2.15.1-reindex-nocase.js @@ -17,6 +17,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info('[2.15.1 migration] UPGRADE BEGIN: 2.15.1-reindex-nocase ') + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect') + logger.info('[2.15.1 migration] UPGRADE END: 2.15.1-reindex-nocase ') + return + } + // Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0 logger.info('[2.15.1 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') await queryInterface.sequelize.query('REINDEX NOCASE;') diff --git a/server/migrations/v2.17.3-fk-constraints.js b/server/migrations/v2.17.3-fk-constraints.js index 5f8a5c9a6..3ad61e585 100644 --- a/server/migrations/v2.17.3-fk-constraints.js +++ b/server/migrations/v2.17.3-fk-constraints.js @@ -18,6 +18,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info('[2.17.3 migration] UPGRADE BEGIN: 2.17.3-fk-constraints') + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect') + logger.info('[2.17.3 migration] UPGRADE END: 2.17.3-fk-constraints') + return + } + const execQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize) // Disable foreign key constraints for the next sequence of operations diff --git a/server/migrations/v2.19.1-copy-title-to-library-items.js b/server/migrations/v2.19.1-copy-title-to-library-items.js index 7b75fa855..d62b008ed 100644 --- a/server/migrations/v2.19.1-copy-title-to-library-items.js +++ b/server/migrations/v2.19.1-copy-title-to-library-items.js @@ -25,6 +25,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`) + logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`) + return + } + await addColumn(queryInterface, logger, 'libraryItems', 'title', { type: queryInterface.sequelize.Sequelize.STRING, allowNull: true }) await copyColumn(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId') await addTrigger(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId') @@ -51,6 +57,12 @@ async function down({ context: { queryInterface, logger } }) { // Downward migration script logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`) + logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`) + return + } + await removeIndex(queryInterface, logger, 'libraryItems', ['libraryId', 'mediaType', 'title']) await removeTrigger(queryInterface, logger, 'libraryItems', 'title') await removeColumn(queryInterface, logger, 'libraryItems', 'title') diff --git a/server/migrations/v2.19.4-improve-podcast-queries.js b/server/migrations/v2.19.4-improve-podcast-queries.js index 689795c31..3becd3162 100644 --- a/server/migrations/v2.19.4-improve-podcast-queries.js +++ b/server/migrations/v2.19.4-improve-podcast-queries.js @@ -26,6 +26,12 @@ async function up({ context: { queryInterface, logger } }) { // Upwards migration script logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`) + logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`) + return + } + // Add numEpisodes column to podcasts table await addColumn(queryInterface, logger, 'podcasts', 'numEpisodes', { type: queryInterface.sequelize.Sequelize.INTEGER, allowNull: false, defaultValue: 0 }) @@ -60,6 +66,12 @@ async function down({ context: { queryInterface, logger } }) { // Downward migration script logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`) + logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`) + return + } + // Remove triggers from libraryItems await removeTrigger(queryInterface, logger, 'podcasts', 'title', 'libraryItems', 'title') await removeTrigger(queryInterface, logger, 'podcasts', 'titleIgnorePrefix', 'libraryItems', 'titleIgnorePrefix') diff --git a/test/server/migrations/v2.15.0-series-column-unique.test.js b/test/server/migrations/v2.15.0-series-column-unique.test.js index 7fce7b7a1..0439dbbb5 100644 --- a/test/server/migrations/v2.15.0-series-column-unique.test.js +++ b/test/server/migrations/v2.15.0-series-column-unique.test.js @@ -88,6 +88,23 @@ describe('migration-v2.15.0-series-column-unique', () => { await queryInterface.dropTable('Series') await queryInterface.dropTable('BookSeries') }) + + it('should skip NOCASE reindex on postgres dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + const originalQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize) + const queryStub = sinon.stub(queryInterface.sequelize, 'query').callsFake((sql, options) => { + if (sql === 'REINDEX NOCASE;') { + throw new Error('Unexpected NOCASE reindex in postgres mode') + } + return originalQuery(sql, options) + }) + + await up({ context: { queryInterface, logger: Logger } }) + + expect(queryStub.neverCalledWith('REINDEX NOCASE;')).to.be.true + expect(loggerInfoStub.calledWith(sinon.match('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.be.true + }) + it('upgrade with no duplicate series', async () => { // Add some entries to the Series table using the UUID for the ids await queryInterface.bulkInsert('Series', [ diff --git a/test/server/migrations/v2.15.1-reindex-nocase.test.js b/test/server/migrations/v2.15.1-reindex-nocase.test.js new file mode 100644 index 000000000..50fe89174 --- /dev/null +++ b/test/server/migrations/v2.15.1-reindex-nocase.test.js @@ -0,0 +1,41 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const { up, down } = require('../../../server/migrations/v2.15.1-reindex-nocase') +const Logger = require('../../../server/Logger') + +describe('migration-v2.15.1-reindex-nocase', () => { + afterEach(() => { + sinon.restore() + }) + + it('should skip reindex on non-sqlite dialect', async () => { + const queryStub = sinon.stub().resolves() + const loggerInfoStub = sinon.stub(Logger, 'info') + const queryInterface = { + sequelize: { + getDialect: () => 'postgres', + query: queryStub + } + } + + await up({ context: { queryInterface, logger: Logger } }) + + expect(queryStub.called).to.equal(false) + expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.equal(true) + }) + + it('should log no-op on down migration', async () => { + const loggerInfoStub = sinon.stub(Logger, 'info') + const queryInterface = { + sequelize: { + getDialect: () => 'postgres', + query: sinon.stub().resolves() + } + } + + await down({ context: { queryInterface, logger: Logger } }) + + expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] No action required for downgrade'))).to.equal(true) + }) +}) diff --git a/test/server/migrations/v2.17.3-fk-constraints.test.js b/test/server/migrations/v2.17.3-fk-constraints.test.js index 33be43ce8..975302831 100644 --- a/test/server/migrations/v2.17.3-fk-constraints.test.js +++ b/test/server/migrations/v2.17.3-fk-constraints.test.js @@ -34,6 +34,16 @@ describe('migration-v2.17.3-fk-constraints', () => { await queryInterface.dropAllTables() }) + it('should skip sqlite-specific migration on non-sqlite dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + const queryStub = sinon.stub(queryInterface.sequelize, 'query').resolves([]) + + await up({ context: { queryInterface, logger: Logger } }) + + expect(queryStub.called).to.equal(false) + expect(loggerInfoStub.calledWith(sinon.match('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect'))).to.equal(true) + }) + it('should fix table foreign key constraints', async () => { // Create tables with missing foreign key constraints: libraryItems, feeds, mediaItemShares, playbackSessions, playlistMediaItems, mediaProgresses await queryInterface.sequelize.query('CREATE TABLE `libraryItems` (`id` UUID UNIQUE PRIMARY KEY, `libraryId` UUID REFERENCES `libraries` (`id`), `libraryFolderId` UUID REFERENCES `libraryFolders` (`id`));') diff --git a/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js b/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js index 5b7678567..7e291927d 100644 --- a/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js +++ b/test/server/migrations/v2.19.1-copy-title-to-library-items.test.js @@ -47,6 +47,18 @@ describe('Migration v2.19.1-copy-title-to-library-items', () => { }) describe('up', () => { + it('should skip sqlite-specific migration on non-sqlite dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + + await up({ context: { queryInterface, logger: Logger } }) + + const table = await queryInterface.describeTable('libraryItems') + expect(table).to.have.property('id') + expect(table).to.not.have.property('title') + expect(table).to.not.have.property('titleIgnorePrefix') + expect(loggerInfoStub.calledWith(sinon.match('[2.19.1 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true + }) + it('should copy title and titleIgnorePrefix to libraryItems', async () => { await up({ context: { queryInterface, logger: Logger } }) diff --git a/test/server/migrations/v2.19.4-improve-podcast-queries.test.js b/test/server/migrations/v2.19.4-improve-podcast-queries.test.js index 0ca697d70..e385318ad 100644 --- a/test/server/migrations/v2.19.4-improve-podcast-queries.test.js +++ b/test/server/migrations/v2.19.4-improve-podcast-queries.test.js @@ -75,6 +75,18 @@ describe('Migration v2.19.4-improve-podcast-queries', () => { }) describe('up', () => { + it('should skip sqlite-specific migration on non-sqlite dialect', async () => { + sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres') + + await up({ context: { queryInterface, logger: Logger } }) + + const podcastsTable = await queryInterface.describeTable('podcasts') + const mediaProgressesTable = await queryInterface.describeTable('mediaProgresses') + expect(podcastsTable).to.not.have.property('numEpisodes') + expect(mediaProgressesTable).to.not.have.property('podcastId') + expect(loggerInfoStub.calledWith(sinon.match('[2.19.4 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true + }) + it('should add numEpisodes column to podcasts', async () => { await up({ context: { queryInterface, logger: Logger } })