diff --git a/package-lock.json b/package-lock.json index 08707893d..178e19626 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "audiobookshelf", - "version": "2.32.1", + "version": "2.32.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "audiobookshelf", - "version": "2.32.1", + "version": "2.32.2", "license": "GPL-3.0", "dependencies": { "axios": "^0.27.2", @@ -23,6 +23,8 @@ "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", + "pg": "^8.19.0", + "pg-hstore": "^2.3.4", "semver": "^7.6.3", "sequelize": "^6.35.2", "socket.io": "^4.5.4", @@ -4138,10 +4140,106 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" }, + "node_modules/pg": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.19.0.tgz", + "integrity": "sha512-QIcLGi508BAHkQ3pJNptsFz5WQMlpGbuBGBaIaXsWK8mel2kQ/rThYI+DbgjUvZrIr7MiuEuc9LcChJoEZK1xQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.11.0", + "pg-pool": "^3.12.0", + "pg-protocol": "^1.12.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, "node_modules/pg-connection-string": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", - "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", + "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", + "license": "MIT" + }, + "node_modules/pg-hstore": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz", + "integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==", + "license": "MIT", + "dependencies": { + "underscore": "^1.13.1" + }, + "engines": { + "node": ">= 0.8.x" + } + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.12.0.tgz", + "integrity": "sha512-eIJ0DES8BLaziFHW7VgJEBPi5hg3Nyng5iKpYtj3wbcAUV9A1wLgWiY7ajf/f/oO1wfxt83phXPY8Emztg7ITg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.12.0.tgz", + "integrity": "sha512-uOANXNRACNdElMXJ0tPz6RBM0XQ61nONGAwlt8da5zs/iUOOCLBQOHSXnrC6fMsvtjxbOJrZZl5IScGv+7mpbg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } }, "node_modules/picocolors": { "version": "1.0.0", @@ -4173,6 +4271,45 @@ "node": ">=8" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -4946,6 +5083,15 @@ "node": ">=8" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -5265,6 +5411,12 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "node_modules/underscore": { + "version": "1.13.8", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", + "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", + "license": "MIT" + }, "node_modules/unique-filename": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", @@ -5476,6 +5628,15 @@ "node": ">=4.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 3ee3fb391..87697288c 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "docker": "docker buildx build --platform linux/amd64,linux/arm64 --push . -t advplyr/audiobookshelf", "docker-amd64-local": "docker buildx build --platform linux/amd64 --load . -t advplyr/audiobookshelf-amd64-local", "docker-arm64-local": "docker buildx build --platform linux/arm64 --load . -t advplyr/audiobookshelf-arm64-local", + "migrate-sqlite-to-postgres": "node server/scripts/migrateSqliteToPostgres.js", "deploy-linux": "node deploy/linux", "test": "mocha", "coverage": "nyc mocha" @@ -52,6 +53,8 @@ "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", + "pg": "^8.19.0", + "pg-hstore": "^2.3.4", "semver": "^7.6.3", "sequelize": "^6.35.2", "socket.io": "^4.5.4", diff --git a/server/Database.js b/server/Database.js index 213c2c61b..68f6e2729 100644 --- a/server/Database.js +++ b/server/Database.js @@ -14,6 +14,7 @@ class Database { constructor() { this.sequelize = null this.dbPath = null + this.dialect = 'sqlite' this.isNew = false // New absdatabase.sqlite created this.hasRootUser = false // Used to show initialization page in web ui @@ -162,11 +163,37 @@ class Database { return this.models.device } + /** + * @returns {'sqlite'|'postgres'} + */ + getConfiguredDialect() { + const explicitDialect = process.env.DB_DIALECT?.trim()?.toLowerCase() + if (explicitDialect === 'postgres' || explicitDialect === 'sqlite') { + return explicitDialect + } + + const databaseUrl = process.env.DATABASE_URL || '' + if (databaseUrl.startsWith('postgres://') || databaseUrl.startsWith('postgresql://')) { + return 'postgres' + } + + return 'sqlite' + } + + isSqliteDialect() { + return this.dialect === 'sqlite' + } + + isPostgresDialect() { + return this.dialect === 'postgres' + } + /** * Check if db file exists * @returns {boolean} */ async checkHasDb() { + if (!this.isSqliteDialect()) return true if (!(await fs.pathExists(this.dbPath))) { Logger.info(`[Database] absdatabase.sqlite not found at ${this.dbPath}`) return false @@ -174,20 +201,39 @@ class Database { return true } + /** + * Check if any user tables exist (for networked dialects) + * @returns {Promise} + */ + async checkHasTables() { + const queryInterface = this.sequelize.getQueryInterface() + const tables = await queryInterface.showAllTables() + return Array.isArray(tables) && tables.length > 0 + } + /** * Connect to db, build models and run migrations * @param {boolean} [force=false] Used for testing, drops & re-creates all tables */ async init(force = false) { - this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') - - // First check if this is a new database - this.isNew = !(await this.checkHasDb()) || force + this.dialect = this.getConfiguredDialect() + if (this.isSqliteDialect()) { + this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') + // First check if this is a new database + this.isNew = !(await this.checkHasDb()) || force + } else { + this.dbPath = process.env.DATABASE_URL || null + this.isNew = !!force + } if (!(await this.connect())) { throw new Error('Database connection failed') } + if (this.isPostgresDialect() && !force) { + this.isNew = !(await this.checkHasTables()) + } + try { const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath) await migrationManager.init(packageJson.version) @@ -214,7 +260,11 @@ class Database { * @returns {boolean} */ async connect() { - Logger.info(`[Database] Initializing db at "${this.dbPath}"`) + if (this.isSqliteDialect()) { + Logger.info(`[Database] Initializing sqlite db at "${this.dbPath}"`) + } else { + Logger.info(`[Database] Initializing postgres db connection`) + } let logging = false let benchmark = false @@ -229,13 +279,26 @@ class Database { benchmark = true } - this.sequelize = new Sequelize({ - dialect: 'sqlite', - storage: this.dbPath, - logging: logging, - benchmark: benchmark, - transactionType: 'IMMEDIATE' - }) + if (this.isSqliteDialect()) { + this.sequelize = new Sequelize({ + dialect: 'sqlite', + storage: this.dbPath, + logging: logging, + benchmark: benchmark, + transactionType: 'IMMEDIATE' + }) + } else { + if (!process.env.DATABASE_URL) { + Logger.error(`[Database] DATABASE_URL is required when DB_DIALECT=postgres`) + return false + } + + this.sequelize = new Sequelize(process.env.DATABASE_URL, { + dialect: 'postgres', + logging: logging, + benchmark: benchmark + }) + } // Helper function this.sequelize.uppercaseFirst = (str) => (str ? `${str[0].toUpperCase()}${str.substr(1)}` : '') @@ -243,34 +306,36 @@ class Database { try { await this.sequelize.authenticate() - // Set SQLite pragmas from environment variables - const allowedPragmas = [ - { name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' }, - { name: 'cache_size', env: 'SQLITE_CACHE_SIZE' }, - { name: 'temp_store', env: 'SQLITE_TEMP_STORE' } - ] + if (this.isSqliteDialect()) { + // Set SQLite pragmas from environment variables + const allowedPragmas = [ + { name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' }, + { name: 'cache_size', env: 'SQLITE_CACHE_SIZE' }, + { name: 'temp_store', env: 'SQLITE_TEMP_STORE' } + ] - for (const pragma of allowedPragmas) { - const value = process.env[pragma.env] - if (value !== undefined) { - try { - Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`) - await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`) - const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`) - Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result) - } catch (error) { - Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error) + for (const pragma of allowedPragmas) { + const value = process.env[pragma.env] + if (value !== undefined) { + try { + Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`) + await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`) + const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`) + Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result) + } catch (error) { + Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error) + } } } - } - if (process.env.NUSQLITE3_PATH) { - await this.loadExtension(process.env.NUSQLITE3_PATH) - Logger.info(`[Database] Db supports unaccent and unicode foldings`) - this.supportsUnaccent = true - this.supportsUnicodeFoldings = true + if (process.env.NUSQLITE3_PATH) { + await this.loadExtension(process.env.NUSQLITE3_PATH) + Logger.info(`[Database] Db supports unaccent and unicode foldings`) + this.supportsUnaccent = true + this.supportsUnicodeFoldings = true + } } - Logger.info(`[Database] Db connection was successful`) + Logger.info(`[Database] Db connection was successful (${this.dialect})`) return true } catch (error) { Logger.error(`[Database] Failed to connect to db`, error) @@ -307,7 +372,7 @@ class Database { * Disconnect from db */ async disconnect() { - Logger.info(`[Database] Disconnecting sqlite db`) + Logger.info(`[Database] Disconnecting ${this.dialect} db`) await this.sequelize.close() } @@ -315,7 +380,7 @@ class Database { * Reconnect to db and init */ async reconnect() { - Logger.info(`[Database] Reconnecting sqlite db`) + Logger.info(`[Database] Reconnecting ${this.dialect} db`) await this.init() } @@ -368,7 +433,7 @@ class Database { * Loads most of the data from the database. This is a temporary solution. */ async loadData() { - if (this.isNew && (await dbMigration.checkShouldMigrate())) { + if (this.isSqliteDialect() && this.isNew && (await dbMigration.checkShouldMigrate())) { Logger.info(`[Database] New database was created and old database was detected - migrating old to new`) await dbMigration.migrate(this.models) } @@ -847,6 +912,11 @@ WHERE EXISTS ( * It adds triggers to update libraryItems.title[IgnorePrefix] when (books|podcasts).title[IgnorePrefix] is updated */ async addTriggers() { + if (!this.isSqliteDialect()) { + Logger.info(`[Database] Skipping sqlite-only triggers for dialect ${this.dialect}`) + return + } + await this.addTriggerIfNotExists('books', 'title', 'id', 'libraryItems', 'title', 'mediaId') await this.addTriggerIfNotExists('books', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId') await this.addTriggerIfNotExists('podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId') @@ -954,6 +1024,7 @@ WHERE EXISTS ( this.supportsUnaccent = supportsUnaccent this.query = query this.hasAccents = false + this.dialect = sequelize.getDialect() } /** @@ -989,9 +1060,10 @@ WHERE EXISTS ( */ matchExpression(column) { const pattern = this.sequelize.escape(`%${this.query}%`) - if (!this.supportsUnaccent) return `${column} LIKE ${pattern}` + const likeOperator = this.dialect === 'postgres' ? 'ILIKE' : 'LIKE' + if (!this.supportsUnaccent) return `${column} ${likeOperator} ${pattern}` const normalizedColumn = this.hasAccents ? column : this.normalize(column) - return `${normalizedColumn} LIKE ${pattern}` + return `${normalizedColumn} ${likeOperator} ${pattern}` } } } diff --git a/server/Server.js b/server/Server.js index d6f748a1e..7d51041a8 100644 --- a/server/Server.js +++ b/server/Server.js @@ -477,14 +477,27 @@ class Server { // Remove series from hide from continue listening that no longer exist try { - const users = await Database.sequelize.query(`SELECT u.id, u.username, u.extraData, json_group_array(value) AS seriesIdsToRemove FROM users u, json_each(u.extraData->"seriesHideFromContinueListening") LEFT JOIN series se ON se.id = value WHERE se.id IS NULL GROUP BY u.id;`, { - model: Database.userModel, - type: Sequelize.QueryTypes.SELECT + const users = await Database.userModel.findAll({ + attributes: ['id', 'username', 'extraData'] }) + for (const user of users) { - const extraData = JSON.parse(user.extraData) - const existingSeriesIds = extraData.seriesHideFromContinueListening - const seriesIdsToRemove = JSON.parse(user.dataValues.seriesIdsToRemove) + const extraData = typeof user.extraData === 'string' ? JSON.parse(user.extraData || '{}') : user.extraData || {} + const existingSeriesIds = Array.isArray(extraData.seriesHideFromContinueListening) ? extraData.seriesHideFromContinueListening : [] + if (!existingSeriesIds.length) continue + + const existingSeries = await Database.seriesModel.findAll({ + attributes: ['id'], + where: { + id: { + [Sequelize.Op.in]: existingSeriesIds + } + } + }) + const existingSeriesSet = new Set(existingSeries.map((series) => series.id)) + const seriesIdsToRemove = existingSeriesIds.filter((seriesId) => !existingSeriesSet.has(seriesId)) + if (!seriesIdsToRemove.length) continue + Logger.info(`[Server] Found ${seriesIdsToRemove.length} non-existent series in seriesHideFromContinueListening for user "${user.username}" - Removing (${seriesIdsToRemove.join(',')})`) const newExtraData = { ...extraData, diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 55ef45690..104d21467 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -24,6 +24,7 @@ const libraryFilters = require('../utils/queries/libraryFilters') const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters') const authorFilters = require('../utils/queries/authorFilters') const zipHelpers = require('../utils/zipHelpers') +const { isPostgres, noCaseSortExpression, jsonArrayExpand } = require('../utils/sqlDialectHelpers') /** * @typedef RequestUserObject @@ -1039,9 +1040,9 @@ class LibraryController { let order = undefined const direction = payload.sortDesc ? 'DESC' : 'ASC' if (payload.sortBy === 'name') { - order = [[Sequelize.literal('name COLLATE NOCASE'), direction]] + order = [[Sequelize.literal(noCaseSortExpression('name', Database.sequelize)), direction]] } else if (payload.sortBy === 'lastFirst') { - order = [[Sequelize.literal('lastFirst COLLATE NOCASE'), direction]] + order = [[Sequelize.literal(noCaseSortExpression('lastFirst', Database.sequelize)), direction]] } else if (payload.sortBy === 'addedAt') { order = [['createdAt', direction]] } else if (payload.sortBy === 'updatedAt') { @@ -1338,16 +1339,23 @@ class LibraryController { const fileExt = req.query.ext === 'abs' ? 'abs' : 'json' const metadataFilename = `metadata.${fileExt}` + const metadataFilenameCountQuery = isPostgres(Database.sequelize) + ? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{metadata,filename}' = :metadataFilename)` + : `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = :metadataFilename)` + const libraryItemsWithMetadata = await Database.libraryItemModel.findAll({ attributes: ['id', 'libraryFiles'], where: [ { libraryId: req.library.id }, - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = "${metadataFilename}")`), { + Sequelize.where(Sequelize.literal(metadataFilenameCountQuery), { [Sequelize.Op.gte]: 1 }) - ] + ], + replacements: { + metadataFilename + } }) if (!libraryItemsWithMetadata.length) { Logger.info(`[LibraryController] No ${metadataFilename} files found to remove`) diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index e302038ff..588771318 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -94,20 +94,26 @@ class MigrationManager { // Only proceed with migration if there are migrations to run if (migrationsToRun.length > 0) { + const dialect = typeof this.sequelize.getDialect === 'function' ? this.sequelize.getDialect() : 'sqlite' + const isSqlite = !dialect || dialect === 'sqlite' const originalDbPath = path.join(this.configPath, 'absdatabase.sqlite') const backupDbPath = path.join(this.configPath, 'absdatabase.backup.sqlite') try { Logger.info(`[MigrationManager] Migrating database ${migrationDirection} to version ${this.serverVersion}`) Logger.info(`[MigrationManager] Migrations to run: ${migrationsToRun.join(', ')}`) - // Create a backup copy of the SQLite database before starting migrations - await fs.copy(originalDbPath, backupDbPath) - Logger.info('Created a backup of the original database.') + if (isSqlite) { + // Create a backup copy of the SQLite database before starting migrations + await fs.copy(originalDbPath, backupDbPath) + Logger.info('Created a backup of the original database.') + } // Run migrations await this.umzug[migrationDirection]({ migrations: migrationsToRun, rerun: 'ALLOW' }) - // Clean up the backup - await fs.remove(backupDbPath) + if (isSqlite) { + // Clean up the backup + await fs.remove(backupDbPath) + } Logger.info('[MigrationManager] Migrations successfully applied to the original database.') } catch (error) { @@ -115,13 +121,15 @@ class MigrationManager { await this.sequelize.close() - // Step 3: If migration fails, save the failed original and restore the backup - const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite') - await fs.move(originalDbPath, failedDbPath, { overwrite: true }) - Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.') + if (isSqlite) { + // Step 3: If migration fails, save the failed original and restore the backup + const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite') + await fs.move(originalDbPath, failedDbPath, { overwrite: true }) + Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.') - await fs.move(backupDbPath, originalDbPath, { overwrite: true }) - Logger.info('[MigrationManager] Restored the original database from the backup.') + await fs.move(backupDbPath, originalDbPath, { overwrite: true }) + Logger.info('[MigrationManager] Restored the original database from the backup.') + } Logger.info('[MigrationManager] Migration failed. Exiting Audiobookshelf with code 1.') process.exit(1) @@ -191,16 +199,15 @@ class MigrationManager { } async fetchVersionsFromDatabase() { + const migrationsMetaTable = `"${MigrationManager.MIGRATIONS_META_TABLE}"` await this.checkOrCreateMigrationsMetaTable() - const [{ version }] = await this.sequelize.query("SELECT value as version FROM :migrationsMeta WHERE key = 'version'", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [{ version }] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, { type: Sequelize.QueryTypes.SELECT }) this.databaseVersion = version - const [{ maxVersion }] = await this.sequelize.query("SELECT value as maxVersion FROM :migrationsMeta WHERE key = 'maxVersion'", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [{ maxVersion }] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, { type: Sequelize.QueryTypes.SELECT }) this.maxVersion = maxVersion @@ -212,8 +219,7 @@ class MigrationManager { // If the table exists, check that the `version` and `maxVersion` rows exist if (migrationsMetaTableExists) { - const [{ count }] = await this.sequelize.query("SELECT COUNT(*) as count FROM :migrationsMeta WHERE key IN ('version', 'maxVersion')", { - replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM "${MigrationManager.MIGRATIONS_META_TABLE}" WHERE key IN ('version', 'maxVersion')`, { type: Sequelize.QueryTypes.SELECT }) if (count < 2) { @@ -241,8 +247,8 @@ class MigrationManager { allowNull: false } }) - await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", { - replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`INSERT INTO "${MigrationManager.MIGRATIONS_META_TABLE}" (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, { + replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0' }, type: Sequelize.QueryTypes.INSERT }) Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`) @@ -299,8 +305,8 @@ class MigrationManager { async updateMaxVersion() { try { - await this.sequelize.query("UPDATE :migrationsMeta SET value = :maxVersion WHERE key = 'maxVersion'", { - replacements: { maxVersion: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :maxVersion WHERE key = 'maxVersion'`, { + replacements: { maxVersion: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) } catch (error) { @@ -311,8 +317,8 @@ class MigrationManager { async updateDatabaseVersion() { try { - await this.sequelize.query("UPDATE :migrationsMeta SET value = :version WHERE key = 'version'", { - replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :version WHERE key = 'version'`, { + replacements: { version: this.serverVersion }, type: Sequelize.QueryTypes.UPDATE }) } catch (error) { diff --git a/server/migrations/v2.20.0-improve-author-sort-queries.js b/server/migrations/v2.20.0-improve-author-sort-queries.js index 53016c709..c5c2b8f5a 100644 --- a/server/migrations/v2.20.0-improve-author-sort-queries.js +++ b/server/migrations/v2.20.0-improve-author-sort-queries.js @@ -39,6 +39,11 @@ const authorsJoin = `${authors} JOIN ${bookAuthors} ON ${authors}.id = ${bookAut * @returns {Promise} - A promise that resolves when the migration is complete. */ async function up({ context: { queryInterface, logger } }) { + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`) + return + } + const helper = new MigrationHelper(queryInterface, logger) // Upwards migration script @@ -72,6 +77,11 @@ async function up({ context: { queryInterface, logger } }) { * @returns {Promise} - A promise that resolves when the migration is complete. */ async function down({ context: { queryInterface, logger } }) { + if (queryInterface.sequelize.getDialect() !== 'sqlite') { + logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`) + return + } + // Downward migration script logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) diff --git a/server/models/Author.js b/server/models/Author.js index 287b66976..8185020df 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -128,6 +128,8 @@ class Author extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' } + super.init( { id: { @@ -146,12 +148,7 @@ class Author extends Model { modelName: 'author', indexes: [ { - fields: [ - { - name: 'name', - collate: 'NOCASE' - } - ] + fields: [nameIndexField] }, // { // fields: [{ diff --git a/server/models/Book.js b/server/models/Book.js index 96371f3a2..08982bb03 100644 --- a/server/models/Book.js +++ b/server/models/Book.js @@ -137,6 +137,8 @@ class Book extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' } + super.init( { id: { @@ -171,12 +173,7 @@ class Book extends Model { modelName: 'book', indexes: [ { - fields: [ - { - name: 'title', - collate: 'NOCASE' - } - ] + fields: [titleIndexField] }, // { // fields: [{ diff --git a/server/models/LibraryItem.js b/server/models/LibraryItem.js index 16a521615..22c8b60a2 100644 --- a/server/models/LibraryItem.js +++ b/server/models/LibraryItem.js @@ -662,6 +662,11 @@ class LibraryItem extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' } + const titleIgnorePrefixIndexField = sequelize.getDialect() === 'postgres' ? 'titleIgnorePrefix' : { name: 'titleIgnorePrefix', collate: 'NOCASE' } + const authorNamesFirstLastIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesFirstLast' : { name: 'authorNamesFirstLast', collate: 'NOCASE' } + const authorNamesLastFirstIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesLastFirst' : { name: 'authorNamesLastFirst', collate: 'NOCASE' } + super.init( { id: { @@ -710,16 +715,16 @@ class LibraryItem extends Model { fields: ['libraryId', 'mediaType', 'createdAt'] }, { - fields: ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', titleIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', titleIgnorePrefixIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', authorNamesFirstLastIndexField] }, { - fields: ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }] + fields: ['libraryId', 'mediaType', authorNamesLastFirstIndexField] }, { fields: ['libraryId', 'mediaId', 'mediaType'] diff --git a/server/models/Series.js b/server/models/Series.js index 6ca288464..28450a010 100644 --- a/server/models/Series.js +++ b/server/models/Series.js @@ -87,6 +87,8 @@ class Series extends Model { * @param {import('../Database').sequelize} sequelize */ static init(sequelize) { + const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' } + super.init( { id: { @@ -103,12 +105,7 @@ class Series extends Model { modelName: 'series', indexes: [ { - fields: [ - { - name: 'name', - collate: 'NOCASE' - } - ] + fields: [nameIndexField] }, // { // fields: [{ diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 640c82d76..f0ef43fec 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -14,6 +14,7 @@ const LibraryItemScanner = require('./LibraryItemScanner') const LibraryScan = require('./LibraryScan') const LibraryItemScanData = require('./LibraryItemScanData') const Task = require('../objects/Task') +const { isPostgres, jsonArrayExpand } = require('../utils/sqlDialectHelpers') class LibraryScanner { constructor() { @@ -674,12 +675,16 @@ async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingle // check if it was moved from another folder by comparing the ino to the library files const ino = await fileUtils.getIno(fullPath) if (!ino) return null + const inodeMatchQuery = isPostgres(Database.sequelize) + ? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{ino}' = :inode)` + : `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)` + const existingLibraryItem = await Database.libraryItemModel.findOneExpanded( [ { libraryId: libraryId }, - sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), { + sequelize.where(sequelize.literal(inodeMatchQuery), { [sequelize.Op.gt]: 0 }) ], diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js new file mode 100644 index 000000000..60acc11e2 --- /dev/null +++ b/server/scripts/migrateSqliteToPostgres.js @@ -0,0 +1,243 @@ +#!/usr/bin/env node + +const sqlite3 = require('sqlite3') +const { Client } = require('pg') + +const SQLITE_PATH = process.env.SQLITE_PATH || '/config/absdatabase.sqlite' +const DATABASE_URL = process.env.DATABASE_URL +const PG_SCHEMA = process.env.PG_SCHEMA || 'public' +const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500) +const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true' + +if (!DATABASE_URL) { + console.error('[migrate] DATABASE_URL is required') + process.exit(1) +} + +const preferredOrder = [ + 'migrationsMeta', + 'SequelizeMeta', + 'settings', + 'users', + 'apiKeys', + 'sessions', + 'libraries', + 'libraryFolders', + 'authors', + 'series', + 'books', + 'podcasts', + 'podcastEpisodes', + 'libraryItems', + 'bookAuthors', + 'bookSeries', + 'collections', + 'collectionBooks', + 'playlists', + 'playlistMediaItems', + 'mediaProgresses', + 'devices', + 'playbackSessions', + 'feeds', + 'feedEpisodes', + 'mediaItemShares', + 'customMetadataProviders' +] + +function quoteIdent(identifier) { + return `"${String(identifier).replace(/"/g, '""')}"` +} + +function openSqlite(filePath) { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(filePath, sqlite3.OPEN_READONLY, (err) => { + if (err) return reject(err) + resolve(db) + }) + }) +} + +function sqliteAll(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => { + if (err) return reject(err) + resolve(rows) + }) + }) +} + +function sqliteGet(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.get(sql, params, (err, row) => { + if (err) return reject(err) + resolve(row) + }) + }) +} + +function normalizeBoolean(value) { + if (value === null || value === undefined) return null + if (typeof value === 'boolean') return value + if (typeof value === 'number') return value !== 0 + if (typeof value === 'string') { + const v = value.trim().toLowerCase() + return v === 'true' || v === '1' || v === 't' + } + return !!value +} + +function normalizeJson(value) { + if (value === null || value === undefined || value === '') return null + if (typeof value === 'object') return value + try { + return JSON.parse(value) + } catch (error) { + return value + } +} + +function convertValue(value, pgColumn) { + if (!pgColumn) return value + const dataType = pgColumn.data_type + const udtName = pgColumn.udt_name + + if (dataType === 'boolean') { + return normalizeBoolean(value) + } + + if (dataType === 'json' || dataType === 'jsonb' || udtName === 'json' || udtName === 'jsonb') { + return normalizeJson(value) + } + + return value +} + +async function main() { + console.log(`[migrate] sqlite source: ${SQLITE_PATH}`) + console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`) + console.log(`[migrate] dry run: ${DRY_RUN}`) + + const sqliteDb = await openSqlite(SQLITE_PATH) + const pg = new Client({ connectionString: DATABASE_URL }) + await pg.connect() + + try { + const sqliteTablesRows = await sqliteAll( + sqliteDb, + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + ) + const sqliteTables = sqliteTablesRows.map((row) => row.name) + + const pgTablesRows = await pg.query( + `SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`, + [PG_SCHEMA] + ) + const pgTables = new Set(pgTablesRows.rows.map((row) => row.table_name)) + + const tablesToMigrate = sqliteTables.filter((table) => pgTables.has(table)) + tablesToMigrate.sort((a, b) => { + const ai = preferredOrder.indexOf(a) + const bi = preferredOrder.indexOf(b) + if (ai === -1 && bi === -1) return a.localeCompare(b) + if (ai === -1) return 1 + if (bi === -1) return -1 + return ai - bi + }) + + if (!tablesToMigrate.length) { + throw new Error('No overlapping tables found between SQLite and PostgreSQL') + } + + console.log(`[migrate] tables to migrate: ${tablesToMigrate.join(', ')}`) + + const pgColumnsByTable = new Map() + for (const table of tablesToMigrate) { + const columnsResult = await pg.query( + `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, + [PG_SCHEMA, table] + ) + const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name, column])) + pgColumnsByTable.set(table, columnMap) + } + + if (!DRY_RUN) { + await pg.query('BEGIN') + await pg.query('SET session_replication_role = replica') + + const truncateList = tablesToMigrate.map((table) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`).join(', ') + await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`) + console.log('[migrate] truncated target tables') + } + + for (const table of tablesToMigrate) { + const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(table)}`) + const pgColumns = pgColumnsByTable.get(table) + const insertColumns = rows.length ? Object.keys(rows[0]).filter((column) => pgColumns.has(column)) : [] + + if (!rows.length || !insertColumns.length) { + console.log(`[migrate] ${table}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`) + continue + } + + if (!DRY_RUN) { + for (let offset = 0; offset < rows.length; offset += BATCH_SIZE) { + const batchRows = rows.slice(offset, offset + BATCH_SIZE) + const valuesSql = [] + const params = [] + let paramIndex = 1 + + for (const row of batchRows) { + const placeholders = [] + for (const column of insertColumns) { + const pgColumn = pgColumns.get(column) + params.push(convertValue(row[column], pgColumn)) + placeholders.push(`$${paramIndex++}`) + } + valuesSql.push(`(${placeholders.join(', ')})`) + } + + const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)} (${insertColumns.map(quoteIdent).join(', ')}) VALUES ${valuesSql.join(', ')}` + await pg.query(insertSql, params) + } + } + + console.log(`[migrate] ${table}: ${rows.length} rows`) + } + + if (!DRY_RUN) { + await pg.query('SET session_replication_role = DEFAULT') + await pg.query('COMMIT') + console.log('[migrate] migration transaction committed') + } + + const parity = [] + for (const table of tablesToMigrate) { + const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(table)}`) + const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`) + parity.push({ + table, + sqliteCount: Number(sqliteCountRow.count || 0), + postgresCount: Number(pgCountResult.rows[0].count || 0) + }) + } + + const mismatches = parity.filter((row) => row.sqliteCount !== row.postgresCount) + if (mismatches.length) { + console.error('[migrate] row-count mismatches detected:') + mismatches.forEach((row) => { + console.error(`[migrate] ${row.table}: sqlite=${row.sqliteCount} postgres=${row.postgresCount}`) + }) + process.exitCode = 2 + } else { + console.log('[migrate] parity check passed for all migrated tables') + } + } finally { + sqliteDb.close() + await pg.end() + } +} + +main().catch((error) => { + console.error('[migrate] failed:', error) + process.exit(1) +}) diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index 7312b9d5d..49f5381e1 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -5,6 +5,7 @@ const libraryItemsBookFilters = require('./libraryItemsBookFilters') const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters') const { createNewSortInstance } = require('../../libs/fastSort') const { profile } = require('../../utils/profiler') +const { booleanLiteral, jsonArrayContainsAny } = require('../sqlDialectHelpers') const naturalSort = createNewSortInstance({ comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare }) @@ -235,13 +236,13 @@ module.exports = { if (userPermissionBookWhere.bookWhere.length) { let attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id' if (!user.canAccessExplicitContent) { - attrQuery += ' AND b.explicit = 0' + attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}` } if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (user.permissions.selectedTagsNotAccessible) { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0` } else { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0` } } seriesWhere.push( diff --git a/server/utils/queries/libraryItemFilters.js b/server/utils/queries/libraryItemFilters.js index 7f95d0ecc..15c3dedb7 100644 --- a/server/utils/queries/libraryItemFilters.js +++ b/server/utils/queries/libraryItemFilters.js @@ -2,6 +2,7 @@ const Sequelize = require('sequelize') const Database = require('../../Database') const libraryItemsBookFilters = require('./libraryItemsBookFilters') const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters') +const { jsonArrayContainsAny } = require('../sqlDialectHelpers') module.exports = { /** @@ -12,7 +13,7 @@ module.exports = { async getAllLibraryItemsWithTags(tags) { const libraryItems = [] const booksWithTag = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -46,7 +47,7 @@ module.exports = { libraryItems.push(libraryItem) } const podcastsWithTag = await Database.podcastModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -77,7 +78,7 @@ module.exports = { async getAllLibraryItemsWithGenres(genres) { const libraryItems = [] const booksWithGenre = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -107,7 +108,7 @@ module.exports = { libraryItems.push(libraryItem) } const podcastsWithGenre = await Database.podcastModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { @@ -138,7 +139,7 @@ module.exports = { async getAllLibraryItemsWithNarrators(narrators) { const libraryItems = [] const booksWithGenre = await Database.bookModel.findAll({ - where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(narrators) WHERE json_valid(narrators) AND json_each.value IN (:narrators))`), { + where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('narrators', 'narrators', Database.sequelize)), { [Sequelize.Op.gte]: 1 }), replacements: { diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 7ae1dc866..029448529 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -6,6 +6,7 @@ const authorFilters = require('./authorFilters') const ShareManager = require('../../managers/ShareManager') const { profile } = require('../profiler') const stringifySequelizeQuery = require('../stringifySequelizeQuery') +const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand } = require('../sqlDialectHelpers') const countCache = new Map() module.exports = { @@ -27,10 +28,10 @@ module.exports = { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { replacements['userTagsSelected'] = user.permissions.itemTagsSelected if (user.permissions.selectedTagsNotAccessible) { - bookWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0)) + bookWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0)) } else { bookWhere.push( - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), { + Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) ) @@ -189,7 +190,7 @@ module.exports = { } else if (group === 'explicit') { mediaWhere['explicit'] = true } else if (['genres', 'tags', 'narrators'].includes(group)) { - mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), { + mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) replacements.filterValue = value @@ -256,9 +257,9 @@ module.exports = { const getTitleOrder = () => { if (global.ServerSettings.sortingIgnorePrefix) { - return [Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir] + return [Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir] } else { - return [Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir] + return [Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir] } } @@ -273,20 +274,23 @@ module.exports = { } else if (sortBy === 'media.duration') { return [['duration', dir]] } else if (sortBy === 'media.metadata.publishedYear') { - return [[Sequelize.literal(`CAST(\`book\`.\`publishedYear\` AS INTEGER)`), dir]] + return [[Sequelize.literal('CAST(book.publishedYear AS INTEGER)'), dir]] } else if (sortBy === 'media.metadata.authorNameLF') { // Sort by author name last first, secondary sort by title - return [[Sequelize.literal('`libraryItem`.`authorNamesLastFirst` COLLATE NOCASE'), dir], getTitleOrder()] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesLastFirst', Database.sequelize)), dir], getTitleOrder()] } else if (sortBy === 'media.metadata.authorName') { // Sort by author name first last, secondary sort by title - return [[Sequelize.literal('`libraryItem`.`authorNamesFirstLast` COLLATE NOCASE'), dir], getTitleOrder()] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesFirstLast', Database.sequelize)), dir], getTitleOrder()] } else if (sortBy === 'media.metadata.title') { if (collapseseries) { - return [[Sequelize.literal('display_title COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('display_title', Database.sequelize)), dir]] } return [getTitleOrder()] } else if (sortBy === 'sequence') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' + if (Database.sequelize.getDialect() === 'postgres') { + return [[Sequelize.literal(`CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ${nullDir}`)]] + } return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]] } else if (sortBy === 'progress') { return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]] @@ -326,7 +330,13 @@ module.exports = { required: true } ], - order: [Sequelize.literal('CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')] + order: [ + Sequelize.literal( + Database.sequelize.getDialect() === 'postgres' + ? 'CAST("books.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST' + : 'CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST' + ) + ] }) const bookSeriesToInclude = [] const booksToInclude = [] @@ -455,12 +465,28 @@ module.exports = { }) } else if (filterGroup === 'ebooks' && filterValue === 'supplementary') { // TODO: Temp workaround for filtering supplementary ebook - libraryItemWhere['libraryFiles'] = { - [Sequelize.Op.substring]: `"isSupplementary":true` + if (Database.sequelize.getDialect() === 'postgres') { + libraryItemWhere[Sequelize.Op.and] = [ + Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), { + [Sequelize.Op.like]: '%"isSupplementary":true%' + }) + ] + } else { + libraryItemWhere['libraryFiles'] = { + [Sequelize.Op.substring]: `"isSupplementary":true` + } } } else if (filterGroup === 'ebooks' && filterValue === 'no-supplementary') { - libraryItemWhere['libraryFiles'] = { - [Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`) + if (Database.sequelize.getDialect() === 'postgres') { + libraryItemWhere[Sequelize.Op.and] = [ + Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), { + [Sequelize.Op.notLike]: '%"isSupplementary":true%' + }) + ] + } else { + libraryItemWhere['libraryFiles'] = { + [Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`) + } } } else if (filterGroup === 'missing' && filterValue === 'authors') { authorInclude = { @@ -502,7 +528,13 @@ module.exports = { }) if (sortBy !== 'sequence') { // Secondary sort by sequence - sortOrder.push([Sequelize.literal('CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]) + sortOrder.push([ + Sequelize.literal( + Database.sequelize.getDialect() === 'postgres' + ? 'CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST' + : 'CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST' + ) + ]) } } else if (filterGroup === 'issues') { libraryItemWhere[Sequelize.Op.or] = [ @@ -594,10 +626,14 @@ module.exports = { // When collapsing series and sorting by title then use the series name instead of the book title // for this set an attribute "display_title" to use in sorting + const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."title"' : '`libraryItem`.`title`' + const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."titleIgnorePrefix"' : '`libraryItem`.`titleIgnorePrefix`' + const fallbackFn = coalesceFunctionName(Database.sequelize) + if (global.ServerSettings.sortingIgnorePrefix) { - bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`titleIgnorePrefix\`)`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title']) } else { - bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`title\`)`), 'display_title']) + bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitle})`), 'display_title']) } } @@ -716,13 +752,14 @@ module.exports = { bookWhere.push(...userPermissionBookWhere.bookWhere) let includeAttributes = [[Sequelize.literal('(SELECT max(mp.updatedAt) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id)'), 'recent_progress']] - let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = 0 OR mp.isFinished IS NULL)` + let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = ${booleanLiteral(false, Database.sequelize)} OR mp.isFinished IS NULL)` if (library.settings.onlyShowLaterBooksInContinueSeries) { - const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as FLOAT) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = 1 AND mp.userId = :userId AND bs.seriesId = series.id)` + const castType = Database.sequelize.getDialect() === 'postgres' ? 'DOUBLE PRECISION' : 'FLOAT' + const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as ${castType}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)` includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence']) - booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as FLOAT) > ${maxSequenceQuery}` + booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as ${castType}) > ${maxSequenceQuery}` } const { rows: series, count } = await Database.seriesModel.findAndCountAll({ @@ -735,7 +772,7 @@ module.exports = { }, // TODO: Simplify queries // Has at least 1 book finished - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = 1)`), { + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)})`), { [Sequelize.Op.gte]: 1 }), // Has at least 1 book not finished (that has a sequence number higher than the highest already read, if library config is toggled) @@ -743,7 +780,7 @@ module.exports = { [Sequelize.Op.gte]: 1 }), // Has no books in progress - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = 0 AND mp.currentTime > 0)`), 0) + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)} AND mp.currentTime > 0)`), 0) ], attributes: { include: includeAttributes @@ -760,7 +797,7 @@ module.exports = { order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], where: { '$book.mediaProgresses.isFinished$': { - [Sequelize.Op.or]: [null, 0] + [Sequelize.Op.or]: [null, false] } }, include: { @@ -854,7 +891,7 @@ module.exports = { { libraryId }, - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = 1 OR mp.currentTime > 0))`), 0) + Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0))`), 0) ], replacements: { userId: user.id, @@ -904,6 +941,7 @@ module.exports = { id: { [Sequelize.Op.in]: booksFromSeriesToInclude } + } ] }, @@ -1130,7 +1168,7 @@ module.exports = { // Search narrators const narratorMatches = [] - const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, { + const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, ${jsonArrayExpand('b.narrators', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1147,7 +1185,7 @@ module.exports = { // Search tags const tagMatches = [] - const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.tags', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1164,7 +1202,7 @@ module.exports = { // Search genres const genreMatches = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -1244,7 +1282,7 @@ module.exports = { */ async getGenresWithCount(libraryId) { const genres = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { replacements: { libraryId }, diff --git a/server/utils/queries/libraryItemsPodcastFilters.js b/server/utils/queries/libraryItemsPodcastFilters.js index 8bb5dc110..d89a4bd73 100644 --- a/server/utils/queries/libraryItemsPodcastFilters.js +++ b/server/utils/queries/libraryItemsPodcastFilters.js @@ -3,6 +3,7 @@ const Database = require('../../Database') const Logger = require('../../Logger') const { profile } = require('../../utils/profiler') const stringifySequelizeQuery = require('../stringifySequelizeQuery') +const { jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression, jsonArrayExpand, jsonPathNumber } = require('../sqlDialectHelpers') const countCache = new Map() @@ -24,10 +25,10 @@ module.exports = { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { replacements['userTagsSelected'] = user.permissions.itemTagsSelected if (user.permissions.selectedTagsNotAccessible) { - podcastWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0)) + podcastWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0)) } else { podcastWhere.push( - Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), { + Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) ) @@ -53,7 +54,7 @@ module.exports = { const replacements = {} if (['genres', 'tags'].includes(group)) { - mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), { + mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), { [Sequelize.Op.gte]: 1 }) replacements.filterValue = value @@ -87,12 +88,12 @@ module.exports = { return [[Sequelize.literal('libraryItem.mtime'), dir]] } else if (sortBy === 'media.metadata.author') { const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' - return [[Sequelize.literal(`\`podcast\`.\`author\` COLLATE NOCASE ${nullDir}`)]] + return [[Sequelize.literal(`${noCaseSortExpression('podcast.author', Database.sequelize)} ${nullDir}`)]] } else if (sortBy === 'media.metadata.title') { if (global.ServerSettings.sortingIgnorePrefix) { - return [[Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]] } else { - return [[Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]] + return [[Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]] } } else if (sortBy === 'media.numTracks') { return [['numEpisodes', dir]] @@ -455,7 +456,7 @@ module.exports = { // Search tags const tagMatches = [] - const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.tags', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -472,7 +473,7 @@ module.exports = { // Search genres const genreMatches = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { replacements: { libraryId: library.id, limit, @@ -563,12 +564,12 @@ module.exports = { * @returns {Promise<{ totalSize:number, totalDuration:number, numAudioFiles:number, totalItems:number}>} */ async getPodcastLibraryStats(libraryId) { - const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = "podcast" AND li.libraryId = :libraryId;`, { + const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = 'podcast' AND li.libraryId = :libraryId;`, { replacements: { libraryId } }) - const [statResults] = await Database.sequelize.query(`SELECT SUM(json_extract(pe.audioFile, '$.duration')) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, { + const [statResults] = await Database.sequelize.query(`SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, { replacements: { libraryId } @@ -588,7 +589,7 @@ module.exports = { */ async getGenresWithCount(libraryId) { const genres = [] - const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { + const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { replacements: { libraryId }, @@ -611,7 +612,7 @@ module.exports = { */ async getLongestPodcasts(libraryId, limit) { const podcasts = await Database.podcastModel.findAll({ - attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(json_extract(pe.audioFile, '$.duration')) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']], + attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']], include: { model: Database.libraryItemModel, attributes: ['id', 'libraryId'], diff --git a/server/utils/queries/seriesFilters.js b/server/utils/queries/seriesFilters.js index ed71e5b3f..3aa208552 100644 --- a/server/utils/queries/seriesFilters.js +++ b/server/utils/queries/seriesFilters.js @@ -2,6 +2,7 @@ const Sequelize = require('sequelize') const Logger = require('../../Logger') const Database = require('../../Database') const libraryItemsBookFilters = require('./libraryItemsBookFilters') +const { booleanLiteral, jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression } = require('../sqlDialectHelpers') module.exports = { decode(text) { @@ -60,7 +61,7 @@ module.exports = { // TODO: Simplify and break-out let attrQuery = null if (['genres', 'tags', 'narrators'].includes(filterGroup)) { - attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (SELECT count(*) FROM json_each(b.${filterGroup}) WHERE json_valid(b.${filterGroup}) AND json_each.value = :filterValue) > 0` + attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ${jsonArrayContainsValue(`b.${filterGroup}`, 'filterValue', Database.sequelize)} > 0` userPermissionBookWhere.replacements.filterValue = filterValue } else if (filterGroup === 'authors') { attrQuery = 'SELECT count(*) FROM books b, bookSeries bs, bookAuthors ba WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ba.bookId = b.id AND ba.authorId = :filterValue' @@ -73,18 +74,18 @@ module.exports = { userPermissionBookWhere.replacements.filterValue = filterValue } else if (filterGroup === 'progress') { if (filterValue === 'not-finished') { - attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)' + attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})` userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'finished') { - const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)' + const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})` seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0)) userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'not-started') { - const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = 1 OR mp.currentTime > 0)' + const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0)` seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0)) userPermissionBookWhere.replacements.userId = user.id } else if (filterValue === 'in-progress') { - attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = 0' + attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)}` userPermissionBookWhere.replacements.userId = user.id } } @@ -95,13 +96,13 @@ module.exports = { if (!attrQuery) attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id' if (!user.canAccessExplicitContent) { - attrQuery += ' AND b.explicit = 0' + attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}` } if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (user.permissions.selectedTagsNotAccessible) { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0` } else { - attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0' + attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0` } } } @@ -128,9 +129,9 @@ module.exports = { order.push(['createdAt', dir]) } else if (sortBy === 'name') { if (global.ServerSettings.sortingIgnorePrefix) { - order.push([Sequelize.literal('nameIgnorePrefix COLLATE NOCASE'), dir]) + order.push([Sequelize.literal(noCaseSortExpression('nameIgnorePrefix', Database.sequelize)), dir]) } else { - order.push([Sequelize.literal('`series`.`name` COLLATE NOCASE'), dir]) + order.push([Sequelize.literal(noCaseSortExpression('series.name', Database.sequelize)), dir]) } } else if (sortBy === 'totalDuration') { seriesAttributes.include.push([Sequelize.literal('(SELECT SUM(b.duration) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND b.id = bs.bookId)'), 'totalDuration']) diff --git a/server/utils/sqlDialectHelpers.js b/server/utils/sqlDialectHelpers.js new file mode 100644 index 000000000..36538bcb1 --- /dev/null +++ b/server/utils/sqlDialectHelpers.js @@ -0,0 +1,75 @@ +function getDialect(sequelize) { + if (!sequelize) return 'sqlite' + if (typeof sequelize.getDialect === 'function') return sequelize.getDialect() + return sequelize.dialect?.name || 'sqlite' +} + +function isPostgres(sequelize) { + return getDialect(sequelize) === 'postgres' +} + +function booleanLiteral(value, sequelize) { + if (isPostgres(sequelize)) return value ? 'TRUE' : 'FALSE' + return value ? '1' : '0' +} + +function noCaseSortExpression(columnExpression, sequelize) { + if (isPostgres(sequelize)) return `LOWER(${columnExpression})` + return `${columnExpression} COLLATE NOCASE` +} + +function coalesceFunctionName(sequelize) { + return isPostgres(sequelize) ? 'COALESCE' : 'IFNULL' +} + +function jsonArrayContainsAny(columnExpression, bindName, sequelize) { + if (isPostgres(sequelize)) { + return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:${bindName}))` + } + return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value IN (:${bindName}))` +} + +function jsonArrayContainsValue(columnExpression, bindName, sequelize) { + if (isPostgres(sequelize)) { + return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :${bindName})` + } + return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value = :${bindName})` +} + +function jsonArrayExpand(columnExpression, sequelize, options = {}) { + const alias = options.alias || 'json_each' + const textValues = options.textValues !== false + if (isPostgres(sequelize)) { + const fn = textValues ? 'jsonb_array_elements_text' : 'jsonb_array_elements' + return `${fn}(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS ${alias}(value)` + } + return `json_each(${columnExpression})` +} + +function jsonPathText(columnExpression, pathSegments, sequelize) { + const path = Array.isArray(pathSegments) ? pathSegments : [pathSegments] + if (isPostgres(sequelize)) { + return `${columnExpression}::jsonb #>> '{${path.join(',')}}'` + } + return `json_extract(${columnExpression}, '$.${path.join('.')}')` +} + +function jsonPathNumber(columnExpression, pathSegments, sequelize) { + if (isPostgres(sequelize)) { + return `NULLIF(${jsonPathText(columnExpression, pathSegments, sequelize)}, '')::double precision` + } + return `json_extract(${columnExpression}, '$.${[].concat(pathSegments).join('.')}')` +} + +module.exports = { + getDialect, + isPostgres, + booleanLiteral, + noCaseSortExpression, + coalesceFunctionName, + jsonArrayContainsAny, + jsonArrayContainsValue, + jsonArrayExpand, + jsonPathText, + jsonPathNumber +} diff --git a/test/server/utils/sqlDialectHelpers.test.js b/test/server/utils/sqlDialectHelpers.test.js new file mode 100644 index 000000000..006b02364 --- /dev/null +++ b/test/server/utils/sqlDialectHelpers.test.js @@ -0,0 +1,58 @@ +const { expect } = require('chai') + +const { + booleanLiteral, + noCaseSortExpression, + coalesceFunctionName, + jsonArrayContainsAny, + jsonArrayContainsValue, + jsonArrayExpand, + jsonPathText, + jsonPathNumber +} = require('../../../server/utils/sqlDialectHelpers') + +const sqlite = { + getDialect: () => 'sqlite' +} + +const postgres = { + getDialect: () => 'postgres' +} + +describe('sqlDialectHelpers', () => { + it('should return sqlite and postgres boolean literals', () => { + expect(booleanLiteral(true, sqlite)).to.equal('1') + expect(booleanLiteral(false, sqlite)).to.equal('0') + expect(booleanLiteral(true, postgres)).to.equal('TRUE') + expect(booleanLiteral(false, postgres)).to.equal('FALSE') + }) + + it('should generate case-insensitive sort expressions', () => { + expect(noCaseSortExpression('name', sqlite)).to.equal('name COLLATE NOCASE') + expect(noCaseSortExpression('name', postgres)).to.equal('LOWER(name)') + }) + + it('should choose the correct null-coalescing function name', () => { + expect(coalesceFunctionName(sqlite)).to.equal('IFNULL') + expect(coalesceFunctionName(postgres)).to.equal('COALESCE') + }) + + it('should generate array membership count queries per dialect', () => { + expect(jsonArrayContainsAny('tags', 'selectedTags', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:selectedTags))') + expect(jsonArrayContainsAny('tags', 'selectedTags', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:selectedTags))") + + expect(jsonArrayContainsValue('tags', 'tag', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value = :tag)') + expect(jsonArrayContainsValue('tags', 'tag', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :tag)") + }) + + it('should generate json array expansion and path extraction by dialect', () => { + expect(jsonArrayExpand('books.tags', sqlite)).to.equal('json_each(books.tags)') + expect(jsonArrayExpand('books.tags', postgres)).to.equal("jsonb_array_elements_text(COALESCE(books.tags::jsonb, '[]'::jsonb)) AS json_each(value)") + + expect(jsonPathText('payload', ['metadata', 'filename'], sqlite)).to.equal("json_extract(payload, '$.metadata.filename')") + expect(jsonPathText('payload', ['metadata', 'filename'], postgres)).to.equal("payload::jsonb #>> '{metadata,filename}'") + + expect(jsonPathNumber('payload', ['duration'], sqlite)).to.equal("json_extract(payload, '$.duration')") + expect(jsonPathNumber('payload', ['duration'], postgres)).to.equal("NULLIF(payload::jsonb #>> '{duration}', '')::double precision") + }) +})