mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2026-07-07 01:41:35 +00:00
Merge 024f892902 into 972193b193
This commit is contained in:
Kevin GATERA
GitHub
commit
5485f3bd44
41 changed files with 2132 additions and 192 deletions
171
package-lock.json
generated
171
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "audiobookshelf",
|
||||
"version": "2.32.1",
|
||||
"version": "2.32.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "audiobookshelf",
|
||||
"version": "2.32.1",
|
||||
"version": "2.32.2",
|
||||
"license": "GPL-3.0",
|
||||
"dependencies": {
|
||||
"axios": "^0.27.2",
|
||||
|
|
@ -23,6 +23,8 @@
|
|||
"p-throttle": "^4.1.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-jwt": "^4.0.1",
|
||||
"pg": "^8.19.0",
|
||||
"pg-hstore": "^2.3.4",
|
||||
"semver": "^7.6.3",
|
||||
"sequelize": "^6.35.2",
|
||||
"socket.io": "^4.5.4",
|
||||
|
|
@ -4138,10 +4140,106 @@
|
|||
"resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz",
|
||||
"integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg=="
|
||||
},
|
||||
"node_modules/pg": {
|
||||
"version": "8.19.0",
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-8.19.0.tgz",
|
||||
"integrity": "sha512-QIcLGi508BAHkQ3pJNptsFz5WQMlpGbuBGBaIaXsWK8mel2kQ/rThYI+DbgjUvZrIr7MiuEuc9LcChJoEZK1xQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"pg-connection-string": "^2.11.0",
|
||||
"pg-pool": "^3.12.0",
|
||||
"pg-protocol": "^1.12.0",
|
||||
"pg-types": "2.2.0",
|
||||
"pgpass": "1.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 16.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"pg-cloudflare": "^1.3.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"pg-native": ">=3.0.1"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"pg-native": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/pg-cloudflare": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz",
|
||||
"integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==",
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/pg-connection-string": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz",
|
||||
"integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg=="
|
||||
"version": "2.11.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz",
|
||||
"integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pg-hstore": {
|
||||
"version": "2.3.4",
|
||||
"resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz",
|
||||
"integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"underscore": "^1.13.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.8.x"
|
||||
}
|
||||
},
|
||||
"node_modules/pg-int8": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
|
||||
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pg-pool": {
|
||||
"version": "3.12.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.12.0.tgz",
|
||||
"integrity": "sha512-eIJ0DES8BLaziFHW7VgJEBPi5hg3Nyng5iKpYtj3wbcAUV9A1wLgWiY7ajf/f/oO1wfxt83phXPY8Emztg7ITg==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"pg": ">=8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pg-protocol": {
|
||||
"version": "1.12.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.12.0.tgz",
|
||||
"integrity": "sha512-uOANXNRACNdElMXJ0tPz6RBM0XQ61nONGAwlt8da5zs/iUOOCLBQOHSXnrC6fMsvtjxbOJrZZl5IScGv+7mpbg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pg-types": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
|
||||
"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"pg-int8": "1.0.1",
|
||||
"postgres-array": "~2.0.0",
|
||||
"postgres-bytea": "~1.0.0",
|
||||
"postgres-date": "~1.0.4",
|
||||
"postgres-interval": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/pgpass": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
|
||||
"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"split2": "^4.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.0.0",
|
||||
|
|
@ -4173,6 +4271,45 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-array": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
|
||||
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-bytea": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
|
||||
"integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-date": {
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
|
||||
"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-interval": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
|
||||
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"xtend": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/prebuild-install": {
|
||||
"version": "7.1.3",
|
||||
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
|
||||
|
|
@ -4946,6 +5083,15 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/split2": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
|
||||
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">= 10.x"
|
||||
}
|
||||
},
|
||||
"node_modules/sprintf-js": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
|
||||
|
|
@ -5265,6 +5411,12 @@
|
|||
"integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/underscore": {
|
||||
"version": "1.13.8",
|
||||
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz",
|
||||
"integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/unique-filename": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz",
|
||||
|
|
@ -5476,6 +5628,15 @@
|
|||
"node": ">=4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/xtend": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
|
||||
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/y18n": {
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
"docker": "docker buildx build --platform linux/amd64,linux/arm64 --push . -t advplyr/audiobookshelf",
|
||||
"docker-amd64-local": "docker buildx build --platform linux/amd64 --load . -t advplyr/audiobookshelf-amd64-local",
|
||||
"docker-arm64-local": "docker buildx build --platform linux/arm64 --load . -t advplyr/audiobookshelf-arm64-local",
|
||||
"migrate-sqlite-to-postgres": "node server/scripts/migrateSqliteToPostgres.js",
|
||||
"deploy-linux": "node deploy/linux",
|
||||
"test": "mocha",
|
||||
"coverage": "nyc mocha"
|
||||
|
|
@ -52,6 +53,8 @@
|
|||
"p-throttle": "^4.1.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-jwt": "^4.0.1",
|
||||
"pg": "^8.19.0",
|
||||
"pg-hstore": "^2.3.4",
|
||||
"semver": "^7.6.3",
|
||||
"sequelize": "^6.35.2",
|
||||
"socket.io": "^4.5.4",
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ class Database {
|
|||
constructor() {
|
||||
this.sequelize = null
|
||||
this.dbPath = null
|
||||
this.dialect = 'sqlite'
|
||||
this.isNew = false // New absdatabase.sqlite created
|
||||
this.hasRootUser = false // Used to show initialization page in web ui
|
||||
|
||||
|
|
@ -162,11 +163,37 @@ class Database {
|
|||
return this.models.device
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {'sqlite'|'postgres'}
|
||||
*/
|
||||
getConfiguredDialect() {
|
||||
const explicitDialect = process.env.DB_DIALECT?.trim()?.toLowerCase()
|
||||
if (explicitDialect === 'postgres' || explicitDialect === 'sqlite') {
|
||||
return explicitDialect
|
||||
}
|
||||
|
||||
const databaseUrl = process.env.DATABASE_URL || ''
|
||||
if (databaseUrl.startsWith('postgres://') || databaseUrl.startsWith('postgresql://')) {
|
||||
return 'postgres'
|
||||
}
|
||||
|
||||
return 'sqlite'
|
||||
}
|
||||
|
||||
isSqliteDialect() {
|
||||
return this.dialect === 'sqlite'
|
||||
}
|
||||
|
||||
isPostgresDialect() {
|
||||
return this.dialect === 'postgres'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if db file exists
|
||||
* @returns {boolean}
|
||||
*/
|
||||
async checkHasDb() {
|
||||
if (!this.isSqliteDialect()) return true
|
||||
if (!(await fs.pathExists(this.dbPath))) {
|
||||
Logger.info(`[Database] absdatabase.sqlite not found at ${this.dbPath}`)
|
||||
return false
|
||||
|
|
@ -174,20 +201,39 @@ class Database {
|
|||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if any user tables exist (for networked dialects)
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async checkHasTables() {
|
||||
const queryInterface = this.sequelize.getQueryInterface()
|
||||
const tables = await queryInterface.showAllTables()
|
||||
return Array.isArray(tables) && tables.length > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to db, build models and run migrations
|
||||
* @param {boolean} [force=false] Used for testing, drops & re-creates all tables
|
||||
*/
|
||||
async init(force = false) {
|
||||
this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite')
|
||||
|
||||
// First check if this is a new database
|
||||
this.isNew = !(await this.checkHasDb()) || force
|
||||
this.dialect = this.getConfiguredDialect()
|
||||
if (this.isSqliteDialect()) {
|
||||
this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite')
|
||||
// First check if this is a new database
|
||||
this.isNew = !(await this.checkHasDb()) || force
|
||||
} else {
|
||||
this.dbPath = process.env.DATABASE_URL || null
|
||||
this.isNew = !!force
|
||||
}
|
||||
|
||||
if (!(await this.connect())) {
|
||||
throw new Error('Database connection failed')
|
||||
}
|
||||
|
||||
if (this.isPostgresDialect() && !force) {
|
||||
this.isNew = !(await this.checkHasTables())
|
||||
}
|
||||
|
||||
try {
|
||||
const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath)
|
||||
await migrationManager.init(packageJson.version)
|
||||
|
|
@ -214,7 +260,11 @@ class Database {
|
|||
* @returns {boolean}
|
||||
*/
|
||||
async connect() {
|
||||
Logger.info(`[Database] Initializing db at "${this.dbPath}"`)
|
||||
if (this.isSqliteDialect()) {
|
||||
Logger.info(`[Database] Initializing sqlite db at "${this.dbPath}"`)
|
||||
} else {
|
||||
Logger.info(`[Database] Initializing postgres db connection`)
|
||||
}
|
||||
|
||||
let logging = false
|
||||
let benchmark = false
|
||||
|
|
@ -229,13 +279,27 @@ class Database {
|
|||
benchmark = true
|
||||
}
|
||||
|
||||
this.sequelize = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: this.dbPath,
|
||||
logging: logging,
|
||||
benchmark: benchmark,
|
||||
transactionType: 'IMMEDIATE'
|
||||
})
|
||||
if (this.isSqliteDialect()) {
|
||||
this.sequelize = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: this.dbPath,
|
||||
logging: logging,
|
||||
benchmark: benchmark,
|
||||
transactionType: 'IMMEDIATE'
|
||||
})
|
||||
} else {
|
||||
if (!process.env.DATABASE_URL) {
|
||||
Logger.error(`[Database] DATABASE_URL is required when DB_DIALECT=postgres`)
|
||||
return false
|
||||
}
|
||||
|
||||
this.sequelize = new Sequelize(process.env.DATABASE_URL, {
|
||||
dialect: 'postgres',
|
||||
logging: logging,
|
||||
benchmark: benchmark,
|
||||
quoteIdentifiers: false
|
||||
})
|
||||
}
|
||||
|
||||
// Helper function
|
||||
this.sequelize.uppercaseFirst = (str) => (str ? `${str[0].toUpperCase()}${str.substr(1)}` : '')
|
||||
|
|
@ -243,34 +307,36 @@ class Database {
|
|||
try {
|
||||
await this.sequelize.authenticate()
|
||||
|
||||
// Set SQLite pragmas from environment variables
|
||||
const allowedPragmas = [
|
||||
{ name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' },
|
||||
{ name: 'cache_size', env: 'SQLITE_CACHE_SIZE' },
|
||||
{ name: 'temp_store', env: 'SQLITE_TEMP_STORE' }
|
||||
]
|
||||
if (this.isSqliteDialect()) {
|
||||
// Set SQLite pragmas from environment variables
|
||||
const allowedPragmas = [
|
||||
{ name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' },
|
||||
{ name: 'cache_size', env: 'SQLITE_CACHE_SIZE' },
|
||||
{ name: 'temp_store', env: 'SQLITE_TEMP_STORE' }
|
||||
]
|
||||
|
||||
for (const pragma of allowedPragmas) {
|
||||
const value = process.env[pragma.env]
|
||||
if (value !== undefined) {
|
||||
try {
|
||||
Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`)
|
||||
await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`)
|
||||
const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`)
|
||||
Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result)
|
||||
} catch (error) {
|
||||
Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error)
|
||||
for (const pragma of allowedPragmas) {
|
||||
const value = process.env[pragma.env]
|
||||
if (value !== undefined) {
|
||||
try {
|
||||
Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`)
|
||||
await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`)
|
||||
const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`)
|
||||
Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result)
|
||||
} catch (error) {
|
||||
Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env.NUSQLITE3_PATH) {
|
||||
await this.loadExtension(process.env.NUSQLITE3_PATH)
|
||||
Logger.info(`[Database] Db supports unaccent and unicode foldings`)
|
||||
this.supportsUnaccent = true
|
||||
this.supportsUnicodeFoldings = true
|
||||
if (process.env.NUSQLITE3_PATH) {
|
||||
await this.loadExtension(process.env.NUSQLITE3_PATH)
|
||||
Logger.info(`[Database] Db supports unaccent and unicode foldings`)
|
||||
this.supportsUnaccent = true
|
||||
this.supportsUnicodeFoldings = true
|
||||
}
|
||||
}
|
||||
Logger.info(`[Database] Db connection was successful`)
|
||||
Logger.info(`[Database] Db connection was successful (${this.dialect})`)
|
||||
return true
|
||||
} catch (error) {
|
||||
Logger.error(`[Database] Failed to connect to db`, error)
|
||||
|
|
@ -307,7 +373,7 @@ class Database {
|
|||
* Disconnect from db
|
||||
*/
|
||||
async disconnect() {
|
||||
Logger.info(`[Database] Disconnecting sqlite db`)
|
||||
Logger.info(`[Database] Disconnecting ${this.dialect} db`)
|
||||
await this.sequelize.close()
|
||||
}
|
||||
|
||||
|
|
@ -315,7 +381,7 @@ class Database {
|
|||
* Reconnect to db and init
|
||||
*/
|
||||
async reconnect() {
|
||||
Logger.info(`[Database] Reconnecting sqlite db`)
|
||||
Logger.info(`[Database] Reconnecting ${this.dialect} db`)
|
||||
await this.init()
|
||||
}
|
||||
|
||||
|
|
@ -346,6 +412,11 @@ class Database {
|
|||
require('./models/CustomMetadataProvider').init(this.sequelize)
|
||||
require('./models/MediaItemShare').init(this.sequelize)
|
||||
|
||||
if (this.isPostgresDialect() && !force && !this.isNew) {
|
||||
Logger.info('[Database] Skipping sequelize.sync for existing postgres schema')
|
||||
return Promise.resolve()
|
||||
}
|
||||
|
||||
return this.sequelize.sync({ force, alter: false })
|
||||
}
|
||||
|
||||
|
|
@ -368,7 +439,7 @@ class Database {
|
|||
* Loads most of the data from the database. This is a temporary solution.
|
||||
*/
|
||||
async loadData() {
|
||||
if (this.isNew && (await dbMigration.checkShouldMigrate())) {
|
||||
if (this.isSqliteDialect() && this.isNew && (await dbMigration.checkShouldMigrate())) {
|
||||
Logger.info(`[Database] New database was created and old database was detected - migrating old to new`)
|
||||
await dbMigration.migrate(this.models)
|
||||
}
|
||||
|
|
@ -847,6 +918,11 @@ WHERE EXISTS (
|
|||
* It adds triggers to update libraryItems.title[IgnorePrefix] when (books|podcasts).title[IgnorePrefix] is updated
|
||||
*/
|
||||
async addTriggers() {
|
||||
if (!this.isSqliteDialect()) {
|
||||
Logger.info(`[Database] Skipping sqlite-only triggers for dialect ${this.dialect}`)
|
||||
return
|
||||
}
|
||||
|
||||
await this.addTriggerIfNotExists('books', 'title', 'id', 'libraryItems', 'title', 'mediaId')
|
||||
await this.addTriggerIfNotExists('books', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId')
|
||||
await this.addTriggerIfNotExists('podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId')
|
||||
|
|
@ -954,6 +1030,7 @@ WHERE EXISTS (
|
|||
this.supportsUnaccent = supportsUnaccent
|
||||
this.query = query
|
||||
this.hasAccents = false
|
||||
this.dialect = sequelize.getDialect()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -989,9 +1066,10 @@ WHERE EXISTS (
|
|||
*/
|
||||
matchExpression(column) {
|
||||
const pattern = this.sequelize.escape(`%${this.query}%`)
|
||||
if (!this.supportsUnaccent) return `${column} LIKE ${pattern}`
|
||||
const likeOperator = this.dialect === 'postgres' ? 'ILIKE' : 'LIKE'
|
||||
if (!this.supportsUnaccent) return `${column} ${likeOperator} ${pattern}`
|
||||
const normalizedColumn = this.hasAccents ? column : this.normalize(column)
|
||||
return `${normalizedColumn} LIKE ${pattern}`
|
||||
return `${normalizedColumn} ${likeOperator} ${pattern}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -477,14 +477,27 @@ class Server {
|
|||
|
||||
// Remove series from hide from continue listening that no longer exist
|
||||
try {
|
||||
const users = await Database.sequelize.query(`SELECT u.id, u.username, u.extraData, json_group_array(value) AS seriesIdsToRemove FROM users u, json_each(u.extraData->"seriesHideFromContinueListening") LEFT JOIN series se ON se.id = value WHERE se.id IS NULL GROUP BY u.id;`, {
|
||||
model: Database.userModel,
|
||||
type: Sequelize.QueryTypes.SELECT
|
||||
const users = await Database.userModel.findAll({
|
||||
attributes: ['id', 'username', 'extraData']
|
||||
})
|
||||
|
||||
for (const user of users) {
|
||||
const extraData = JSON.parse(user.extraData)
|
||||
const existingSeriesIds = extraData.seriesHideFromContinueListening
|
||||
const seriesIdsToRemove = JSON.parse(user.dataValues.seriesIdsToRemove)
|
||||
const extraData = typeof user.extraData === 'string' ? JSON.parse(user.extraData || '{}') : user.extraData || {}
|
||||
const existingSeriesIds = Array.isArray(extraData.seriesHideFromContinueListening) ? extraData.seriesHideFromContinueListening : []
|
||||
if (!existingSeriesIds.length) continue
|
||||
|
||||
const existingSeries = await Database.seriesModel.findAll({
|
||||
attributes: ['id'],
|
||||
where: {
|
||||
id: {
|
||||
[Sequelize.Op.in]: existingSeriesIds
|
||||
}
|
||||
}
|
||||
})
|
||||
const existingSeriesSet = new Set(existingSeries.map((series) => series.id))
|
||||
const seriesIdsToRemove = existingSeriesIds.filter((seriesId) => !existingSeriesSet.has(seriesId))
|
||||
if (!seriesIdsToRemove.length) continue
|
||||
|
||||
Logger.info(`[Server] Found ${seriesIdsToRemove.length} non-existent series in seriesHideFromContinueListening for user "${user.username}" - Removing (${seriesIdsToRemove.join(',')})`)
|
||||
const newExtraData = {
|
||||
...extraData,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ const libraryFilters = require('../utils/queries/libraryFilters')
|
|||
const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters')
|
||||
const authorFilters = require('../utils/queries/authorFilters')
|
||||
const zipHelpers = require('../utils/zipHelpers')
|
||||
const { isPostgres, noCaseSortExpression, jsonArrayExpand } = require('../utils/sqlDialectHelpers')
|
||||
|
||||
/**
|
||||
* @typedef RequestUserObject
|
||||
|
|
@ -1039,9 +1040,9 @@ class LibraryController {
|
|||
let order = undefined
|
||||
const direction = payload.sortDesc ? 'DESC' : 'ASC'
|
||||
if (payload.sortBy === 'name') {
|
||||
order = [[Sequelize.literal('name COLLATE NOCASE'), direction]]
|
||||
order = [[Sequelize.literal(noCaseSortExpression('name', Database.sequelize)), direction]]
|
||||
} else if (payload.sortBy === 'lastFirst') {
|
||||
order = [[Sequelize.literal('lastFirst COLLATE NOCASE'), direction]]
|
||||
order = [[Sequelize.literal(noCaseSortExpression('lastFirst', Database.sequelize)), direction]]
|
||||
} else if (payload.sortBy === 'addedAt') {
|
||||
order = [['createdAt', direction]]
|
||||
} else if (payload.sortBy === 'updatedAt') {
|
||||
|
|
@ -1338,16 +1339,23 @@ class LibraryController {
|
|||
|
||||
const fileExt = req.query.ext === 'abs' ? 'abs' : 'json'
|
||||
const metadataFilename = `metadata.${fileExt}`
|
||||
const metadataFilenameCountQuery = isPostgres(Database.sequelize)
|
||||
? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{metadata,filename}' = :metadataFilename)`
|
||||
: `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = :metadataFilename)`
|
||||
|
||||
const libraryItemsWithMetadata = await Database.libraryItemModel.findAll({
|
||||
attributes: ['id', 'libraryFiles'],
|
||||
where: [
|
||||
{
|
||||
libraryId: req.library.id
|
||||
},
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = "${metadataFilename}")`), {
|
||||
Sequelize.where(Sequelize.literal(metadataFilenameCountQuery), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
})
|
||||
]
|
||||
],
|
||||
replacements: {
|
||||
metadataFilename
|
||||
}
|
||||
})
|
||||
if (!libraryItemsWithMetadata.length) {
|
||||
Logger.info(`[LibraryController] No ${metadataFilename} files found to remove`)
|
||||
|
|
|
|||
|
|
@ -94,20 +94,26 @@ class MigrationManager {
|
|||
|
||||
// Only proceed with migration if there are migrations to run
|
||||
if (migrationsToRun.length > 0) {
|
||||
const dialect = typeof this.sequelize.getDialect === 'function' ? this.sequelize.getDialect() : 'sqlite'
|
||||
const isSqlite = !dialect || dialect === 'sqlite'
|
||||
const originalDbPath = path.join(this.configPath, 'absdatabase.sqlite')
|
||||
const backupDbPath = path.join(this.configPath, 'absdatabase.backup.sqlite')
|
||||
try {
|
||||
Logger.info(`[MigrationManager] Migrating database ${migrationDirection} to version ${this.serverVersion}`)
|
||||
Logger.info(`[MigrationManager] Migrations to run: ${migrationsToRun.join(', ')}`)
|
||||
// Create a backup copy of the SQLite database before starting migrations
|
||||
await fs.copy(originalDbPath, backupDbPath)
|
||||
Logger.info('Created a backup of the original database.')
|
||||
if (isSqlite) {
|
||||
// Create a backup copy of the SQLite database before starting migrations
|
||||
await fs.copy(originalDbPath, backupDbPath)
|
||||
Logger.info('Created a backup of the original database.')
|
||||
}
|
||||
|
||||
// Run migrations
|
||||
await this.umzug[migrationDirection]({ migrations: migrationsToRun, rerun: 'ALLOW' })
|
||||
|
||||
// Clean up the backup
|
||||
await fs.remove(backupDbPath)
|
||||
if (isSqlite) {
|
||||
// Clean up the backup
|
||||
await fs.remove(backupDbPath)
|
||||
}
|
||||
|
||||
Logger.info('[MigrationManager] Migrations successfully applied to the original database.')
|
||||
} catch (error) {
|
||||
|
|
@ -115,13 +121,15 @@ class MigrationManager {
|
|||
|
||||
await this.sequelize.close()
|
||||
|
||||
// Step 3: If migration fails, save the failed original and restore the backup
|
||||
const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite')
|
||||
await fs.move(originalDbPath, failedDbPath, { overwrite: true })
|
||||
Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.')
|
||||
if (isSqlite) {
|
||||
// Step 3: If migration fails, save the failed original and restore the backup
|
||||
const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite')
|
||||
await fs.move(originalDbPath, failedDbPath, { overwrite: true })
|
||||
Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.')
|
||||
|
||||
await fs.move(backupDbPath, originalDbPath, { overwrite: true })
|
||||
Logger.info('[MigrationManager] Restored the original database from the backup.')
|
||||
await fs.move(backupDbPath, originalDbPath, { overwrite: true })
|
||||
Logger.info('[MigrationManager] Restored the original database from the backup.')
|
||||
}
|
||||
|
||||
Logger.info('[MigrationManager] Migration failed. Exiting Audiobookshelf with code 1.')
|
||||
process.exit(1)
|
||||
|
|
@ -191,29 +199,27 @@ class MigrationManager {
|
|||
}
|
||||
|
||||
async fetchVersionsFromDatabase() {
|
||||
const migrationsMetaTable = MigrationManager.MIGRATIONS_META_TABLE
|
||||
await this.checkOrCreateMigrationsMetaTable()
|
||||
|
||||
const [{ version }] = await this.sequelize.query("SELECT value as version FROM :migrationsMeta WHERE key = 'version'", {
|
||||
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
const [versionRow] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, {
|
||||
type: Sequelize.QueryTypes.SELECT
|
||||
})
|
||||
this.databaseVersion = version
|
||||
this.databaseVersion = versionRow?.version
|
||||
|
||||
const [{ maxVersion }] = await this.sequelize.query("SELECT value as maxVersion FROM :migrationsMeta WHERE key = 'maxVersion'", {
|
||||
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
const [maxVersionRow] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, {
|
||||
type: Sequelize.QueryTypes.SELECT
|
||||
})
|
||||
this.maxVersion = maxVersion
|
||||
this.maxVersion = maxVersionRow?.maxVersion || maxVersionRow?.maxversion
|
||||
}
|
||||
|
||||
async checkOrCreateMigrationsMetaTable() {
|
||||
const queryInterface = this.sequelize.getQueryInterface()
|
||||
let migrationsMetaTableExists = await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE)
|
||||
let migrationsMetaTableExists = await this.tableExists(MigrationManager.MIGRATIONS_META_TABLE)
|
||||
|
||||
// If the table exists, check that the `version` and `maxVersion` rows exist
|
||||
if (migrationsMetaTableExists) {
|
||||
const [{ count }] = await this.sequelize.query("SELECT COUNT(*) as count FROM :migrationsMeta WHERE key IN ('version', 'maxVersion')", {
|
||||
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM ${MigrationManager.MIGRATIONS_META_TABLE} WHERE key IN ('version', 'maxVersion')`, {
|
||||
type: Sequelize.QueryTypes.SELECT
|
||||
})
|
||||
if (count < 2) {
|
||||
|
|
@ -241,14 +247,28 @@ class MigrationManager {
|
|||
allowNull: false
|
||||
}
|
||||
})
|
||||
await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", {
|
||||
replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
await this.sequelize.query(`INSERT INTO ${MigrationManager.MIGRATIONS_META_TABLE} (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, {
|
||||
replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0' },
|
||||
type: Sequelize.QueryTypes.INSERT
|
||||
})
|
||||
Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`)
|
||||
}
|
||||
}
|
||||
|
||||
async tableExists(tableName) {
|
||||
const queryInterface = this.sequelize.getQueryInterface()
|
||||
if (typeof queryInterface.tableExists === 'function') {
|
||||
return queryInterface.tableExists(tableName)
|
||||
}
|
||||
|
||||
const tables = await queryInterface.showAllTables()
|
||||
return tables.some((table) => {
|
||||
if (typeof table === 'string') return table === tableName
|
||||
if (table?.tableName) return table.tableName === tableName
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
extractVersionFromTag(tag) {
|
||||
if (!tag) return null
|
||||
const versionMatch = tag.match(/^v?(\d+\.\d+\.\d+)/)
|
||||
|
|
@ -299,8 +319,8 @@ class MigrationManager {
|
|||
|
||||
async updateMaxVersion() {
|
||||
try {
|
||||
await this.sequelize.query("UPDATE :migrationsMeta SET value = :maxVersion WHERE key = 'maxVersion'", {
|
||||
replacements: { maxVersion: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :maxVersion WHERE key = 'maxVersion'`, {
|
||||
replacements: { maxVersion: this.serverVersion },
|
||||
type: Sequelize.QueryTypes.UPDATE
|
||||
})
|
||||
} catch (error) {
|
||||
|
|
@ -311,8 +331,8 @@ class MigrationManager {
|
|||
|
||||
async updateDatabaseVersion() {
|
||||
try {
|
||||
await this.sequelize.query("UPDATE :migrationsMeta SET value = :version WHERE key = 'version'", {
|
||||
replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :version WHERE key = 'version'`, {
|
||||
replacements: { version: this.serverVersion },
|
||||
type: Sequelize.QueryTypes.UPDATE
|
||||
})
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -19,8 +19,12 @@ async function up({ context: { queryInterface, logger } }) {
|
|||
logger.info('[2.15.0 migration] UPGRADE BEGIN: 2.15.0-series-column-unique ')
|
||||
|
||||
// Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0
|
||||
logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues')
|
||||
await queryInterface.sequelize.query('REINDEX NOCASE;')
|
||||
if (queryInterface.sequelize.getDialect() === 'sqlite') {
|
||||
logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues')
|
||||
await queryInterface.sequelize.query('REINDEX NOCASE;')
|
||||
} else {
|
||||
logger.info('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect')
|
||||
}
|
||||
|
||||
// Check if the unique index already exists
|
||||
const seriesIndexes = await queryInterface.showIndex('Series')
|
||||
|
|
|
|||
|
|
@ -17,6 +17,12 @@ async function up({ context: { queryInterface, logger } }) {
|
|||
// Upwards migration script
|
||||
logger.info('[2.15.1 migration] UPGRADE BEGIN: 2.15.1-reindex-nocase ')
|
||||
|
||||
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
|
||||
logger.info('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect')
|
||||
logger.info('[2.15.1 migration] UPGRADE END: 2.15.1-reindex-nocase ')
|
||||
return
|
||||
}
|
||||
|
||||
// Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0
|
||||
logger.info('[2.15.1 migration] Reindexing NOCASE indices to fix potential hidden corruption issues')
|
||||
await queryInterface.sequelize.query('REINDEX NOCASE;')
|
||||
|
|
|
|||
|
|
@ -18,6 +18,12 @@ async function up({ context: { queryInterface, logger } }) {
|
|||
// Upwards migration script
|
||||
logger.info('[2.17.3 migration] UPGRADE BEGIN: 2.17.3-fk-constraints')
|
||||
|
||||
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
|
||||
logger.info('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect')
|
||||
logger.info('[2.17.3 migration] UPGRADE END: 2.17.3-fk-constraints')
|
||||
return
|
||||
}
|
||||
|
||||
const execQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize)
|
||||
|
||||
// Disable foreign key constraints for the next sequence of operations
|
||||
|
|
|
|||
|
|
@ -25,6 +25,12 @@ async function up({ context: { queryInterface, logger } }) {
|
|||
// Upwards migration script
|
||||
logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
|
||||
|
||||
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
|
||||
logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`)
|
||||
logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
|
||||
return
|
||||
}
|
||||
|
||||
await addColumn(queryInterface, logger, 'libraryItems', 'title', { type: queryInterface.sequelize.Sequelize.STRING, allowNull: true })
|
||||
await copyColumn(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId')
|
||||
await addTrigger(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId')
|
||||
|
|
@ -51,6 +57,12 @@ async function down({ context: { queryInterface, logger } }) {
|
|||
// Downward migration script
|
||||
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
|
||||
|
||||
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
|
||||
logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`)
|
||||
logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
|
||||
return
|
||||
}
|
||||
|
||||
await removeIndex(queryInterface, logger, 'libraryItems', ['libraryId', 'mediaType', 'title'])
|
||||
await removeTrigger(queryInterface, logger, 'libraryItems', 'title')
|
||||
await removeColumn(queryInterface, logger, 'libraryItems', 'title')
|
||||
|
|
|
|||
|
|
@ -26,6 +26,12 @@ async function up({ context: { queryInterface, logger } }) {
|
|||
// Upwards migration script
|
||||
logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
|
||||
|
||||
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
|
||||
logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`)
|
||||
logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
|
||||
return
|
||||
}
|
||||
|
||||
// Add numEpisodes column to podcasts table
|
||||
await addColumn(queryInterface, logger, 'podcasts', 'numEpisodes', { type: queryInterface.sequelize.Sequelize.INTEGER, allowNull: false, defaultValue: 0 })
|
||||
|
||||
|
|
@ -60,6 +66,12 @@ async function down({ context: { queryInterface, logger } }) {
|
|||
// Downward migration script
|
||||
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
|
||||
|
||||
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
|
||||
logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`)
|
||||
logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
|
||||
return
|
||||
}
|
||||
|
||||
// Remove triggers from libraryItems
|
||||
await removeTrigger(queryInterface, logger, 'podcasts', 'title', 'libraryItems', 'title')
|
||||
await removeTrigger(queryInterface, logger, 'podcasts', 'titleIgnorePrefix', 'libraryItems', 'titleIgnorePrefix')
|
||||
|
|
|
|||
|
|
@ -39,6 +39,11 @@ const authorsJoin = `${authors} JOIN ${bookAuthors} ON ${authors}.id = ${bookAut
|
|||
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
|
||||
*/
|
||||
async function up({ context: { queryInterface, logger } }) {
|
||||
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
|
||||
logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`)
|
||||
return
|
||||
}
|
||||
|
||||
const helper = new MigrationHelper(queryInterface, logger)
|
||||
|
||||
// Upwards migration script
|
||||
|
|
@ -72,6 +77,11 @@ async function up({ context: { queryInterface, logger } }) {
|
|||
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
|
||||
*/
|
||||
async function down({ context: { queryInterface, logger } }) {
|
||||
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
|
||||
logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`)
|
||||
return
|
||||
}
|
||||
|
||||
// Downward migration script
|
||||
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
|
||||
|
||||
|
|
|
|||
|
|
@ -128,6 +128,8 @@ class Author extends Model {
|
|||
* @param {import('../Database').sequelize} sequelize
|
||||
*/
|
||||
static init(sequelize) {
|
||||
const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' }
|
||||
|
||||
super.init(
|
||||
{
|
||||
id: {
|
||||
|
|
@ -146,12 +148,7 @@ class Author extends Model {
|
|||
modelName: 'author',
|
||||
indexes: [
|
||||
{
|
||||
fields: [
|
||||
{
|
||||
name: 'name',
|
||||
collate: 'NOCASE'
|
||||
}
|
||||
]
|
||||
fields: [nameIndexField]
|
||||
},
|
||||
// {
|
||||
// fields: [{
|
||||
|
|
|
|||
|
|
@ -137,6 +137,8 @@ class Book extends Model {
|
|||
* @param {import('../Database').sequelize} sequelize
|
||||
*/
|
||||
static init(sequelize) {
|
||||
const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' }
|
||||
|
||||
super.init(
|
||||
{
|
||||
id: {
|
||||
|
|
@ -146,7 +148,7 @@ class Book extends Model {
|
|||
},
|
||||
title: DataTypes.STRING,
|
||||
titleIgnorePrefix: DataTypes.STRING,
|
||||
subtitle: DataTypes.STRING,
|
||||
subtitle: DataTypes.TEXT,
|
||||
publishedYear: DataTypes.STRING,
|
||||
publishedDate: DataTypes.STRING,
|
||||
publisher: DataTypes.STRING,
|
||||
|
|
@ -171,12 +173,7 @@ class Book extends Model {
|
|||
modelName: 'book',
|
||||
indexes: [
|
||||
{
|
||||
fields: [
|
||||
{
|
||||
name: 'title',
|
||||
collate: 'NOCASE'
|
||||
}
|
||||
]
|
||||
fields: [titleIndexField]
|
||||
},
|
||||
// {
|
||||
// fields: [{
|
||||
|
|
|
|||
|
|
@ -662,6 +662,11 @@ class LibraryItem extends Model {
|
|||
* @param {import('../Database').sequelize} sequelize
|
||||
*/
|
||||
static init(sequelize) {
|
||||
const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' }
|
||||
const titleIgnorePrefixIndexField = sequelize.getDialect() === 'postgres' ? 'titleIgnorePrefix' : { name: 'titleIgnorePrefix', collate: 'NOCASE' }
|
||||
const authorNamesFirstLastIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesFirstLast' : { name: 'authorNamesFirstLast', collate: 'NOCASE' }
|
||||
const authorNamesLastFirstIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesLastFirst' : { name: 'authorNamesLastFirst', collate: 'NOCASE' }
|
||||
|
||||
super.init(
|
||||
{
|
||||
id: {
|
||||
|
|
@ -710,16 +715,16 @@ class LibraryItem extends Model {
|
|||
fields: ['libraryId', 'mediaType', 'createdAt']
|
||||
},
|
||||
{
|
||||
fields: ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }]
|
||||
fields: ['libraryId', 'mediaType', titleIndexField]
|
||||
},
|
||||
{
|
||||
fields: ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }]
|
||||
fields: ['libraryId', 'mediaType', titleIgnorePrefixIndexField]
|
||||
},
|
||||
{
|
||||
fields: ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }]
|
||||
fields: ['libraryId', 'mediaType', authorNamesFirstLastIndexField]
|
||||
},
|
||||
{
|
||||
fields: ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }]
|
||||
fields: ['libraryId', 'mediaType', authorNamesLastFirstIndexField]
|
||||
},
|
||||
{
|
||||
fields: ['libraryId', 'mediaId', 'mediaType']
|
||||
|
|
|
|||
|
|
@ -253,7 +253,13 @@ class MediaProgress extends Model {
|
|||
const escapedDate = this.sequelize.escape(new Date(progressPayload.lastUpdate))
|
||||
Logger.info(`[MediaProgress] Manually setting updatedAt to ${escapedDate} (media item ${this.mediaItemId})`)
|
||||
|
||||
await this.sequelize.query(`UPDATE "mediaProgresses" SET "updatedAt" = ${escapedDate} WHERE "id" = '${this.id}'`)
|
||||
await this.constructor.update(
|
||||
{ updatedAt: new Date(progressPayload.lastUpdate) },
|
||||
{
|
||||
where: { id: this.id },
|
||||
silent: true
|
||||
}
|
||||
)
|
||||
|
||||
await this.reload()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ class PlaybackSession extends Model {
|
|||
currentTime: DataTypes.FLOAT,
|
||||
serverVersion: DataTypes.STRING,
|
||||
coverPath: DataTypes.STRING,
|
||||
timeListening: DataTypes.INTEGER,
|
||||
timeListening: DataTypes.FLOAT,
|
||||
mediaMetadata: DataTypes.JSON,
|
||||
date: DataTypes.STRING,
|
||||
dayOfWeek: DataTypes.STRING,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
const { DataTypes, Model, where, fn, col, literal } = require('sequelize')
|
||||
|
||||
const { getTitlePrefixAtEnd, getTitleIgnorePrefix } = require('../utils/index')
|
||||
const { safeTextToDoubleExpression } = require('../utils/sqlDialectHelpers')
|
||||
|
||||
class Series extends Model {
|
||||
constructor(values, options) {
|
||||
|
|
@ -87,6 +88,8 @@ class Series extends Model {
|
|||
* @param {import('../Database').sequelize} sequelize
|
||||
*/
|
||||
static init(sequelize) {
|
||||
const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' }
|
||||
|
||||
super.init(
|
||||
{
|
||||
id: {
|
||||
|
|
@ -103,12 +106,7 @@ class Series extends Model {
|
|||
modelName: 'series',
|
||||
indexes: [
|
||||
{
|
||||
fields: [
|
||||
{
|
||||
name: 'name',
|
||||
collate: 'NOCASE'
|
||||
}
|
||||
]
|
||||
fields: [nameIndexField]
|
||||
},
|
||||
// {
|
||||
// fields: [{
|
||||
|
|
@ -161,7 +159,7 @@ class Series extends Model {
|
|||
}
|
||||
}
|
||||
],
|
||||
order: [[literal('CAST(`bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]]
|
||||
order: [[literal(`${safeTextToDoubleExpression(this.sequelize.getDialect() === 'postgres' ? '"bookSeries"."sequence"' : '`bookSeries.sequence`', this.sequelize)} ASC NULLS LAST`)]]
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -422,10 +422,24 @@ class User extends Model {
|
|||
const cachedUser = userCache.getById(userId) || userCache.getByOldId(userId)
|
||||
if (cachedUser) return cachedUser
|
||||
|
||||
const isUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userId)
|
||||
if (isUuid) {
|
||||
const byId = await this.findByPk(userId, {
|
||||
include: this.sequelize.models.mediaProgress
|
||||
})
|
||||
if (byId) {
|
||||
userCache.set(byId)
|
||||
return byId
|
||||
}
|
||||
}
|
||||
|
||||
const oldIdMatcher =
|
||||
this.sequelize.getDialect() === 'postgres'
|
||||
? sequelize.where(sequelize.literal(`extradata#>>'{oldUserId}'`), userId)
|
||||
: { 'extraData.oldUserId': userId }
|
||||
|
||||
const user = await this.findOne({
|
||||
where: {
|
||||
[sequelize.Op.or]: [{ id: userId }, { 'extraData.oldUserId': userId }]
|
||||
},
|
||||
where: oldIdMatcher,
|
||||
include: this.sequelize.models.mediaProgress
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ const LibraryItemScanner = require('./LibraryItemScanner')
|
|||
const LibraryScan = require('./LibraryScan')
|
||||
const LibraryItemScanData = require('./LibraryItemScanData')
|
||||
const Task = require('../objects/Task')
|
||||
const { isPostgres, jsonArrayExpand } = require('../utils/sqlDialectHelpers')
|
||||
|
||||
class LibraryScanner {
|
||||
constructor() {
|
||||
|
|
@ -674,12 +675,16 @@ async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingle
|
|||
// check if it was moved from another folder by comparing the ino to the library files
|
||||
const ino = await fileUtils.getIno(fullPath)
|
||||
if (!ino) return null
|
||||
const inodeMatchQuery = isPostgres(Database.sequelize)
|
||||
? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{ino}' = :inode)`
|
||||
: `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)`
|
||||
|
||||
const existingLibraryItem = await Database.libraryItemModel.findOneExpanded(
|
||||
[
|
||||
{
|
||||
libraryId: libraryId
|
||||
},
|
||||
sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), {
|
||||
sequelize.where(sequelize.literal(inodeMatchQuery), {
|
||||
[sequelize.Op.gt]: 0
|
||||
})
|
||||
],
|
||||
|
|
|
|||
490
server/scripts/migrateSqliteToPostgres.js
Normal file
490
server/scripts/migrateSqliteToPostgres.js
Normal file
|
|
@ -0,0 +1,490 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
const sqlite3 = require('sqlite3')
|
||||
const { Client } = require('pg')
|
||||
|
||||
const SQLITE_PATH = process.env.SQLITE_PATH || '/config/absdatabase.sqlite'
|
||||
const DATABASE_URL = process.env.DATABASE_URL
|
||||
const PG_SCHEMA = process.env.PG_SCHEMA || 'public'
|
||||
const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500)
|
||||
const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true'
|
||||
|
||||
const preferredOrder = [
|
||||
'migrationsMeta',
|
||||
'SequelizeMeta',
|
||||
'settings',
|
||||
'users',
|
||||
'apiKeys',
|
||||
'sessions',
|
||||
'libraries',
|
||||
'libraryFolders',
|
||||
'authors',
|
||||
'series',
|
||||
'books',
|
||||
'podcasts',
|
||||
'podcastEpisodes',
|
||||
'libraryItems',
|
||||
'bookAuthors',
|
||||
'bookSeries',
|
||||
'collections',
|
||||
'collectionBooks',
|
||||
'playlists',
|
||||
'playlistMediaItems',
|
||||
'mediaProgresses',
|
||||
'devices',
|
||||
'playbackSessions',
|
||||
'feeds',
|
||||
'feedEpisodes',
|
||||
'mediaItemShares',
|
||||
'customMetadataProviders'
|
||||
]
|
||||
|
||||
function quoteIdent(identifier) {
|
||||
return `"${String(identifier).replace(/"/g, '""')}"`
|
||||
}
|
||||
|
||||
function openSqlite(filePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const db = new sqlite3.Database(filePath, sqlite3.OPEN_READONLY, (err) => {
|
||||
if (err) return reject(err)
|
||||
resolve(db)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function sqliteAll(db, sql, params = []) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.all(sql, params, (err, rows) => {
|
||||
if (err) return reject(err)
|
||||
resolve(rows)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function sqliteGet(db, sql, params = []) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.get(sql, params, (err, row) => {
|
||||
if (err) return reject(err)
|
||||
resolve(row)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) {
|
||||
const issues = []
|
||||
|
||||
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
|
||||
const pgColumns = pgColumnsByTable.get(postgresTable)
|
||||
if (!pgColumns) continue
|
||||
|
||||
for (const column of pgColumns.values()) {
|
||||
if (column.data_type !== 'character varying' || !column.character_maximum_length) continue
|
||||
|
||||
const sqliteColumn = column.column_name
|
||||
const maxLength = Number(column.character_maximum_length)
|
||||
const quotedTable = quoteIdent(sqliteTable)
|
||||
const quotedColumn = quoteIdent(sqliteColumn)
|
||||
|
||||
try {
|
||||
const maxLengthRow = await sqliteGet(
|
||||
sqliteDb,
|
||||
`SELECT MAX(LENGTH(${quotedColumn})) AS maxLength FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`
|
||||
)
|
||||
const actualMaxLength = Number(maxLengthRow?.maxLength || 0)
|
||||
if (actualMaxLength <= maxLength) continue
|
||||
|
||||
const overCountRow = await sqliteGet(
|
||||
sqliteDb,
|
||||
`SELECT COUNT(*) AS count FROM ${quotedTable} WHERE LENGTH(${quotedColumn}) > ?`,
|
||||
[maxLength]
|
||||
)
|
||||
|
||||
issues.push({
|
||||
sqliteTable,
|
||||
sqliteColumn,
|
||||
postgresTable,
|
||||
postgresColumn: column.column_name,
|
||||
maxLength,
|
||||
actualMaxLength,
|
||||
overCount: Number(overCountRow?.count || 0)
|
||||
})
|
||||
} catch (error) {
|
||||
// Ignore columns missing in sqlite source table
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return issues
|
||||
}
|
||||
|
||||
function isIntegerCompatible(value) {
|
||||
if (value === null || value === undefined) return true
|
||||
|
||||
if (typeof value === 'number') {
|
||||
return Number.isFinite(value) && Number.isInteger(value)
|
||||
}
|
||||
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim()
|
||||
if (!trimmed) return false
|
||||
if (!/^-?\d+$/.test(trimmed)) return false
|
||||
const parsed = Number(trimmed)
|
||||
return Number.isFinite(parsed)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) {
|
||||
const issues = []
|
||||
|
||||
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
|
||||
const pgColumns = pgColumnsByTable.get(postgresTable)
|
||||
if (!pgColumns) continue
|
||||
|
||||
for (const column of pgColumns.values()) {
|
||||
const dataType = column.data_type
|
||||
if (dataType !== 'smallint' && dataType !== 'integer' && dataType !== 'bigint') continue
|
||||
|
||||
const sqliteColumn = column.column_name
|
||||
const quotedTable = quoteIdent(sqliteTable)
|
||||
const quotedColumn = quoteIdent(sqliteColumn)
|
||||
|
||||
let rows
|
||||
try {
|
||||
rows = await sqliteAll(sqliteDb, `SELECT ${quotedColumn} AS value FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`)
|
||||
} catch (error) {
|
||||
// Ignore columns missing in sqlite source table
|
||||
continue
|
||||
}
|
||||
|
||||
let badCount = 0
|
||||
let sampleValue = null
|
||||
for (const row of rows) {
|
||||
if (!isIntegerCompatible(row.value)) {
|
||||
badCount += 1
|
||||
if (sampleValue === null) sampleValue = row.value
|
||||
}
|
||||
}
|
||||
|
||||
if (badCount > 0) {
|
||||
issues.push({
|
||||
sqliteTable,
|
||||
sqliteColumn,
|
||||
postgresTable,
|
||||
postgresColumn: column.column_name,
|
||||
postgresType: dataType,
|
||||
badCount,
|
||||
sampleValue
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return issues
|
||||
}
|
||||
|
||||
function normalizeBoolean(value) {
|
||||
if (value === null || value === undefined) return null
|
||||
if (typeof value === 'boolean') return value
|
||||
if (typeof value === 'number') return value !== 0
|
||||
if (typeof value === 'string') {
|
||||
const v = value.trim().toLowerCase()
|
||||
return v === 'true' || v === '1' || v === 't'
|
||||
}
|
||||
return !!value
|
||||
}
|
||||
|
||||
function normalizeJson(value) {
|
||||
if (value === null || value === undefined || value === '') return null
|
||||
if (typeof value === 'object') return JSON.stringify(value)
|
||||
|
||||
const textValue = String(value)
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(textValue)
|
||||
if (typeof parsed === 'string') {
|
||||
try {
|
||||
return JSON.stringify(JSON.parse(parsed))
|
||||
} catch (error) {
|
||||
return JSON.stringify(parsed)
|
||||
}
|
||||
}
|
||||
return JSON.stringify(parsed)
|
||||
} catch (error) {
|
||||
// Keep non-JSON payloads as JSON string values so inserts remain valid JSON.
|
||||
return JSON.stringify(textValue)
|
||||
}
|
||||
}
|
||||
|
||||
function convertValue(value, pgColumn) {
|
||||
if (!pgColumn) return value
|
||||
const dataType = pgColumn.data_type
|
||||
const udtName = pgColumn.udt_name
|
||||
|
||||
if (dataType === 'boolean') {
|
||||
return normalizeBoolean(value)
|
||||
}
|
||||
|
||||
if (dataType === 'json' || dataType === 'jsonb' || udtName === 'json' || udtName === 'jsonb') {
|
||||
return normalizeJson(value)
|
||||
}
|
||||
|
||||
if ((dataType === 'smallint' || dataType === 'integer' || dataType === 'bigint') && value !== null && value !== undefined) {
|
||||
if (typeof value === 'number') return value
|
||||
if (typeof value === 'string' && /^-?\d+$/.test(value.trim())) {
|
||||
return Number(value)
|
||||
}
|
||||
}
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
function getOverlongColumns(row, insertColumns) {
|
||||
const overlong = []
|
||||
|
||||
for (const column of insertColumns) {
|
||||
const maxLength = column.metadata.character_maximum_length
|
||||
if (!maxLength) continue
|
||||
|
||||
const value = row[column.sqliteColumn]
|
||||
if (value === null || value === undefined) continue
|
||||
|
||||
const length = String(value).length
|
||||
if (length > maxLength) {
|
||||
overlong.push({
|
||||
sqliteColumn: column.sqliteColumn,
|
||||
postgresColumn: column.postgresColumn,
|
||||
maxLength,
|
||||
actualLength: length
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return overlong
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!DATABASE_URL) {
|
||||
throw new Error('DATABASE_URL is required')
|
||||
}
|
||||
|
||||
console.log(`[migrate] sqlite source: ${SQLITE_PATH}`)
|
||||
console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`)
|
||||
console.log(`[migrate] dry run: ${DRY_RUN}`)
|
||||
|
||||
const sqliteDb = await openSqlite(SQLITE_PATH)
|
||||
const pg = new Client({ connectionString: DATABASE_URL })
|
||||
await pg.connect()
|
||||
|
||||
try {
|
||||
const sqliteTablesRows = await sqliteAll(
|
||||
sqliteDb,
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
||||
)
|
||||
const sqliteTables = sqliteTablesRows.map((row) => row.name)
|
||||
|
||||
const pgTablesRows = await pg.query(
|
||||
`SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`,
|
||||
[PG_SCHEMA]
|
||||
)
|
||||
|
||||
const pgTablesByLowerName = new Map(pgTablesRows.rows.map((row) => [row.table_name.toLowerCase(), row.table_name]))
|
||||
|
||||
const tablesToMigrate = sqliteTables
|
||||
.map((sqliteTable) => {
|
||||
const postgresTable = pgTablesByLowerName.get(sqliteTable.toLowerCase())
|
||||
if (!postgresTable) return null
|
||||
return {
|
||||
sqliteTable,
|
||||
postgresTable
|
||||
}
|
||||
})
|
||||
.filter(Boolean)
|
||||
|
||||
tablesToMigrate.sort((a, b) => {
|
||||
const ai = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === a.sqliteTable.toLowerCase())
|
||||
const bi = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === b.sqliteTable.toLowerCase())
|
||||
if (ai === -1 && bi === -1) return a.sqliteTable.localeCompare(b.sqliteTable)
|
||||
if (ai === -1) return 1
|
||||
if (bi === -1) return -1
|
||||
return ai - bi
|
||||
})
|
||||
|
||||
if (!tablesToMigrate.length) {
|
||||
throw new Error('No overlapping tables found between SQLite and PostgreSQL')
|
||||
}
|
||||
|
||||
console.log(`[migrate] tables to migrate: ${tablesToMigrate.map((table) => table.sqliteTable).join(', ')}`)
|
||||
|
||||
const pgColumnsByTable = new Map()
|
||||
for (const { postgresTable } of tablesToMigrate) {
|
||||
const columnsResult = await pg.query(
|
||||
`SELECT column_name, data_type, udt_name, character_maximum_length FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`,
|
||||
[PG_SCHEMA, postgresTable]
|
||||
)
|
||||
const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column]))
|
||||
pgColumnsByTable.set(postgresTable, columnMap)
|
||||
}
|
||||
|
||||
const overlongVarcharIssues = await findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable)
|
||||
const integerTypeIssues = await findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable)
|
||||
|
||||
if (overlongVarcharIssues.length) {
|
||||
console.error('[migrate] overlong source values detected for varchar columns:')
|
||||
overlongVarcharIssues.forEach((issue) => {
|
||||
console.error(
|
||||
`[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` +
|
||||
`(max=${issue.maxLength}, actualMax=${issue.actualMaxLength}, overRows=${issue.overCount})`
|
||||
)
|
||||
})
|
||||
throw new Error('Migration aborted to prevent truncation/data loss. Widen target column types first.')
|
||||
}
|
||||
|
||||
if (integerTypeIssues.length) {
|
||||
console.error('[migrate] non-integer source values detected for integer columns:')
|
||||
integerTypeIssues.forEach((issue) => {
|
||||
console.error(
|
||||
`[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` +
|
||||
`(type=${issue.postgresType}, badRows=${issue.badCount}, sample=${JSON.stringify(issue.sampleValue)})`
|
||||
)
|
||||
})
|
||||
throw new Error('Migration aborted to prevent numeric precision loss. Widen target numeric column types first.')
|
||||
}
|
||||
|
||||
if (!DRY_RUN) {
|
||||
await pg.query('BEGIN')
|
||||
await pg.query('SET session_replication_role = replica')
|
||||
|
||||
const truncateList = tablesToMigrate.map(({ postgresTable }) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`).join(', ')
|
||||
await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`)
|
||||
console.log('[migrate] truncated target tables')
|
||||
}
|
||||
|
||||
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
|
||||
const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(sqliteTable)}`)
|
||||
const pgColumns = pgColumnsByTable.get(postgresTable)
|
||||
const insertColumns = rows.length
|
||||
? Object.keys(rows[0])
|
||||
.map((sqliteColumn) => {
|
||||
const pgColumn = pgColumns.get(sqliteColumn.toLowerCase())
|
||||
if (!pgColumn) return null
|
||||
return {
|
||||
sqliteColumn,
|
||||
postgresColumn: pgColumn.column_name,
|
||||
metadata: pgColumn
|
||||
}
|
||||
})
|
||||
.filter(Boolean)
|
||||
: []
|
||||
|
||||
if (!rows.length || !insertColumns.length) {
|
||||
console.log(`[migrate] ${sqliteTable}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`)
|
||||
continue
|
||||
}
|
||||
|
||||
if (!DRY_RUN) {
|
||||
for (let offset = 0; offset < rows.length; offset += BATCH_SIZE) {
|
||||
const batchRows = rows.slice(offset, offset + BATCH_SIZE)
|
||||
const valuesSql = []
|
||||
const params = []
|
||||
let paramIndex = 1
|
||||
|
||||
for (const row of batchRows) {
|
||||
const placeholders = []
|
||||
for (const column of insertColumns) {
|
||||
params.push(convertValue(row[column.sqliteColumn], column.metadata))
|
||||
placeholders.push(`$${paramIndex++}`)
|
||||
}
|
||||
valuesSql.push(`(${placeholders.join(', ')})`)
|
||||
}
|
||||
|
||||
const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}`
|
||||
|
||||
try {
|
||||
await pg.query('SAVEPOINT migrate_batch')
|
||||
await pg.query(insertSql, params)
|
||||
await pg.query('RELEASE SAVEPOINT migrate_batch')
|
||||
} catch (error) {
|
||||
await pg.query('ROLLBACK TO SAVEPOINT migrate_batch')
|
||||
console.error(`[migrate] batch insert failed for ${sqliteTable} (offset=${offset}, size=${batchRows.length}): ${error.message}`)
|
||||
|
||||
for (let rowIndex = 0; rowIndex < batchRows.length; rowIndex++) {
|
||||
const row = batchRows[rowIndex]
|
||||
const singleRowParams = insertColumns.map((column) => convertValue(row[column.sqliteColumn], column.metadata))
|
||||
const singleRowInsertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES (${singleRowParams.map((_, index) => `$${index + 1}`).join(', ')})`
|
||||
|
||||
try {
|
||||
await pg.query('SAVEPOINT migrate_row')
|
||||
await pg.query(singleRowInsertSql, singleRowParams)
|
||||
await pg.query('RELEASE SAVEPOINT migrate_row')
|
||||
} catch (rowError) {
|
||||
await pg.query('ROLLBACK TO SAVEPOINT migrate_row')
|
||||
const overlongColumns = getOverlongColumns(row, insertColumns)
|
||||
if (overlongColumns.length) {
|
||||
overlongColumns.forEach((column) => {
|
||||
console.error(
|
||||
`[migrate] overlong value in ${sqliteTable}.${column.sqliteColumn} -> ${postgresTable}.${column.postgresColumn} ` +
|
||||
`(length=${column.actualLength}, max=${column.maxLength})`
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
throw rowError
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[migrate] ${sqliteTable}: ${rows.length} rows`)
|
||||
}
|
||||
|
||||
if (!DRY_RUN) {
|
||||
await pg.query('SET session_replication_role = DEFAULT')
|
||||
await pg.query('COMMIT')
|
||||
console.log('[migrate] migration transaction committed')
|
||||
}
|
||||
|
||||
const parity = []
|
||||
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
|
||||
const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(sqliteTable)}`)
|
||||
const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`)
|
||||
parity.push({
|
||||
table: sqliteTable,
|
||||
sqliteCount: Number(sqliteCountRow.count || 0),
|
||||
postgresCount: Number(pgCountResult.rows[0].count || 0)
|
||||
})
|
||||
}
|
||||
|
||||
const mismatches = parity.filter((row) => row.sqliteCount !== row.postgresCount)
|
||||
if (mismatches.length) {
|
||||
console.error('[migrate] row-count mismatches detected:')
|
||||
mismatches.forEach((row) => {
|
||||
console.error(`[migrate] ${row.table}: sqlite=${row.sqliteCount} postgres=${row.postgresCount}`)
|
||||
})
|
||||
process.exitCode = 2
|
||||
} else {
|
||||
console.log('[migrate] parity check passed for all migrated tables')
|
||||
}
|
||||
} finally {
|
||||
sqliteDb.close()
|
||||
await pg.end()
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
console.error('[migrate] failed:', error)
|
||||
process.exit(1)
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
normalizeJson,
|
||||
isIntegerCompatible,
|
||||
convertValue,
|
||||
findOverlongVarcharValues,
|
||||
findIntegerTypeIssues,
|
||||
quoteIdent
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ const libraryItemsBookFilters = require('./libraryItemsBookFilters')
|
|||
const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters')
|
||||
const { createNewSortInstance } = require('../../libs/fastSort')
|
||||
const { profile } = require('../../utils/profiler')
|
||||
const { booleanLiteral, jsonArrayContainsAny } = require('../sqlDialectHelpers')
|
||||
const naturalSort = createNewSortInstance({
|
||||
comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
|
||||
})
|
||||
|
|
@ -235,13 +236,13 @@ module.exports = {
|
|||
if (userPermissionBookWhere.bookWhere.length) {
|
||||
let attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id'
|
||||
if (!user.canAccessExplicitContent) {
|
||||
attrQuery += ' AND b.explicit = 0'
|
||||
attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}`
|
||||
}
|
||||
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
|
||||
if (user.permissions.selectedTagsNotAccessible) {
|
||||
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0'
|
||||
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0`
|
||||
} else {
|
||||
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0'
|
||||
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0`
|
||||
}
|
||||
}
|
||||
seriesWhere.push(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ const Sequelize = require('sequelize')
|
|||
const Database = require('../../Database')
|
||||
const libraryItemsBookFilters = require('./libraryItemsBookFilters')
|
||||
const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters')
|
||||
const { jsonArrayContainsAny } = require('../sqlDialectHelpers')
|
||||
|
||||
module.exports = {
|
||||
/**
|
||||
|
|
@ -12,7 +13,7 @@ module.exports = {
|
|||
async getAllLibraryItemsWithTags(tags) {
|
||||
const libraryItems = []
|
||||
const booksWithTag = await Database.bookModel.findAll({
|
||||
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), {
|
||||
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
replacements: {
|
||||
|
|
@ -46,7 +47,7 @@ module.exports = {
|
|||
libraryItems.push(libraryItem)
|
||||
}
|
||||
const podcastsWithTag = await Database.podcastModel.findAll({
|
||||
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), {
|
||||
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
replacements: {
|
||||
|
|
@ -77,7 +78,7 @@ module.exports = {
|
|||
async getAllLibraryItemsWithGenres(genres) {
|
||||
const libraryItems = []
|
||||
const booksWithGenre = await Database.bookModel.findAll({
|
||||
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), {
|
||||
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
replacements: {
|
||||
|
|
@ -107,7 +108,7 @@ module.exports = {
|
|||
libraryItems.push(libraryItem)
|
||||
}
|
||||
const podcastsWithGenre = await Database.podcastModel.findAll({
|
||||
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), {
|
||||
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
replacements: {
|
||||
|
|
@ -138,7 +139,7 @@ module.exports = {
|
|||
async getAllLibraryItemsWithNarrators(narrators) {
|
||||
const libraryItems = []
|
||||
const booksWithGenre = await Database.bookModel.findAll({
|
||||
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(narrators) WHERE json_valid(narrators) AND json_each.value IN (:narrators))`), {
|
||||
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('narrators', 'narrators', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
replacements: {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ const authorFilters = require('./authorFilters')
|
|||
const ShareManager = require('../../managers/ShareManager')
|
||||
const { profile } = require('../profiler')
|
||||
const stringifySequelizeQuery = require('../stringifySequelizeQuery')
|
||||
const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand, safeTextToDoubleExpression, safeTextToIntegerExpression } = require('../sqlDialectHelpers')
|
||||
const countCache = new Map()
|
||||
|
||||
module.exports = {
|
||||
|
|
@ -27,10 +28,10 @@ module.exports = {
|
|||
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
|
||||
replacements['userTagsSelected'] = user.permissions.itemTagsSelected
|
||||
if (user.permissions.selectedTagsNotAccessible) {
|
||||
bookWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0))
|
||||
bookWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0))
|
||||
} else {
|
||||
bookWhere.push(
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), {
|
||||
Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
})
|
||||
)
|
||||
|
|
@ -189,7 +190,7 @@ module.exports = {
|
|||
} else if (group === 'explicit') {
|
||||
mediaWhere['explicit'] = true
|
||||
} else if (['genres', 'tags', 'narrators'].includes(group)) {
|
||||
mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), {
|
||||
mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
})
|
||||
replacements.filterValue = value
|
||||
|
|
@ -236,7 +237,7 @@ module.exports = {
|
|||
} else if (group === 'publishedDecades') {
|
||||
const startYear = parseInt(value)
|
||||
const endYear = parseInt(value, 10) + 9
|
||||
mediaWhere = Sequelize.where(Sequelize.literal('CAST(publishedYear AS INTEGER)'), {
|
||||
mediaWhere = Sequelize.where(Sequelize.literal(safeTextToIntegerExpression('publishedYear', Database.sequelize)), {
|
||||
[Sequelize.Op.between]: [startYear, endYear]
|
||||
})
|
||||
}
|
||||
|
|
@ -256,9 +257,9 @@ module.exports = {
|
|||
|
||||
const getTitleOrder = () => {
|
||||
if (global.ServerSettings.sortingIgnorePrefix) {
|
||||
return [Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]
|
||||
return [Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]
|
||||
} else {
|
||||
return [Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]
|
||||
return [Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -273,21 +274,22 @@ module.exports = {
|
|||
} else if (sortBy === 'media.duration') {
|
||||
return [['duration', dir]]
|
||||
} else if (sortBy === 'media.metadata.publishedYear') {
|
||||
return [[Sequelize.literal(`CAST(\`book\`.\`publishedYear\` AS INTEGER)`), dir]]
|
||||
return [[Sequelize.literal(safeTextToIntegerExpression('book.publishedYear', Database.sequelize)), dir]]
|
||||
} else if (sortBy === 'media.metadata.authorNameLF') {
|
||||
// Sort by author name last first, secondary sort by title
|
||||
return [[Sequelize.literal('`libraryItem`.`authorNamesLastFirst` COLLATE NOCASE'), dir], getTitleOrder()]
|
||||
return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesLastFirst', Database.sequelize)), dir], getTitleOrder()]
|
||||
} else if (sortBy === 'media.metadata.authorName') {
|
||||
// Sort by author name first last, secondary sort by title
|
||||
return [[Sequelize.literal('`libraryItem`.`authorNamesFirstLast` COLLATE NOCASE'), dir], getTitleOrder()]
|
||||
return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesFirstLast', Database.sequelize)), dir], getTitleOrder()]
|
||||
} else if (sortBy === 'media.metadata.title') {
|
||||
if (collapseseries) {
|
||||
return [[Sequelize.literal('display_title COLLATE NOCASE'), dir]]
|
||||
return [[Sequelize.literal(noCaseSortExpression('display_title', Database.sequelize)), dir]]
|
||||
}
|
||||
return [getTitleOrder()]
|
||||
} else if (sortBy === 'sequence') {
|
||||
const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'
|
||||
return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]]
|
||||
const sequenceColumn = Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`'
|
||||
return [[Sequelize.literal(`${safeTextToDoubleExpression(sequenceColumn, Database.sequelize)} ${nullDir}`)]]
|
||||
} else if (sortBy === 'progress') {
|
||||
return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]]
|
||||
} else if (sortBy === 'progress.createdAt') {
|
||||
|
|
@ -326,7 +328,9 @@ module.exports = {
|
|||
required: true
|
||||
}
|
||||
],
|
||||
order: [Sequelize.literal('CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]
|
||||
order: [
|
||||
Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"books->bookSeries"."sequence"' : '`books.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`)
|
||||
]
|
||||
})
|
||||
const bookSeriesToInclude = []
|
||||
const booksToInclude = []
|
||||
|
|
@ -358,26 +362,32 @@ module.exports = {
|
|||
|
||||
async findAndCountAll(findOptions, limit, offset, useCountCache) {
|
||||
const model = Database.bookModel
|
||||
if (useCountCache) {
|
||||
const countCacheKey = stringifySequelizeQuery(findOptions)
|
||||
Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`)
|
||||
if (!countCache.has(countCacheKey)) {
|
||||
const count = await model.count(findOptions)
|
||||
countCache.set(countCacheKey, count)
|
||||
}
|
||||
|
||||
findOptions.limit = limit || null
|
||||
findOptions.offset = offset
|
||||
|
||||
const rows = await model.findAll(findOptions)
|
||||
|
||||
return { rows, count: countCache.get(countCacheKey) }
|
||||
}
|
||||
|
||||
findOptions.limit = limit || null
|
||||
findOptions.offset = offset
|
||||
|
||||
return await model.findAndCountAll(findOptions)
|
||||
try {
|
||||
if (useCountCache) {
|
||||
const countCacheKey = stringifySequelizeQuery(findOptions)
|
||||
Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`)
|
||||
if (!countCache.has(countCacheKey)) {
|
||||
const count = await model.count(findOptions)
|
||||
countCache.set(countCacheKey, count)
|
||||
}
|
||||
|
||||
const rows = await model.findAll(findOptions)
|
||||
return { rows, count: countCache.get(countCacheKey) }
|
||||
}
|
||||
|
||||
return await model.findAndCountAll(findOptions)
|
||||
} catch (error) {
|
||||
Logger.error(`[LibraryItemsBookFilters] findAndCountAll failed: ${error.message}`)
|
||||
try {
|
||||
Logger.error(`[LibraryItemsBookFilters] findAndCountAll query: ${stringifySequelizeQuery(findOptions)}`)
|
||||
} catch (stringifyError) {
|
||||
Logger.error(`[LibraryItemsBookFilters] failed to stringify query: ${stringifyError.message}`)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
|
|
@ -455,12 +465,28 @@ module.exports = {
|
|||
})
|
||||
} else if (filterGroup === 'ebooks' && filterValue === 'supplementary') {
|
||||
// TODO: Temp workaround for filtering supplementary ebook
|
||||
libraryItemWhere['libraryFiles'] = {
|
||||
[Sequelize.Op.substring]: `"isSupplementary":true`
|
||||
if (Database.sequelize.getDialect() === 'postgres') {
|
||||
libraryItemWhere[Sequelize.Op.and] = [
|
||||
Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), {
|
||||
[Sequelize.Op.like]: '%"isSupplementary":true%'
|
||||
})
|
||||
]
|
||||
} else {
|
||||
libraryItemWhere['libraryFiles'] = {
|
||||
[Sequelize.Op.substring]: `"isSupplementary":true`
|
||||
}
|
||||
}
|
||||
} else if (filterGroup === 'ebooks' && filterValue === 'no-supplementary') {
|
||||
libraryItemWhere['libraryFiles'] = {
|
||||
[Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`)
|
||||
if (Database.sequelize.getDialect() === 'postgres') {
|
||||
libraryItemWhere[Sequelize.Op.and] = [
|
||||
Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), {
|
||||
[Sequelize.Op.notLike]: '%"isSupplementary":true%'
|
||||
})
|
||||
]
|
||||
} else {
|
||||
libraryItemWhere['libraryFiles'] = {
|
||||
[Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`)
|
||||
}
|
||||
}
|
||||
} else if (filterGroup === 'missing' && filterValue === 'authors') {
|
||||
authorInclude = {
|
||||
|
|
@ -502,7 +528,9 @@ module.exports = {
|
|||
})
|
||||
if (sortBy !== 'sequence') {
|
||||
// Secondary sort by sequence
|
||||
sortOrder.push([Sequelize.literal('CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')])
|
||||
sortOrder.push([
|
||||
Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`)
|
||||
])
|
||||
}
|
||||
} else if (filterGroup === 'issues') {
|
||||
libraryItemWhere[Sequelize.Op.or] = [
|
||||
|
|
@ -594,10 +622,32 @@ module.exports = {
|
|||
|
||||
// When collapsing series and sorting by title then use the series name instead of the book title
|
||||
// for this set an attribute "display_title" to use in sorting
|
||||
const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.title' : '`libraryItem`.`title`'
|
||||
const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.titleIgnorePrefix' : '`libraryItem`.`titleIgnorePrefix`'
|
||||
const fallbackFn = coalesceFunctionName(Database.sequelize)
|
||||
const includedBookSeriesIds = bookSeriesToInclude.map((v) => Database.sequelize.escape(v.id)).join(', ')
|
||||
const collapseSeriesSubqueryByName = includedBookSeriesIds
|
||||
? `(SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))`
|
||||
: 'NULL'
|
||||
const collapseSeriesSubqueryByNameIgnorePrefix = includedBookSeriesIds
|
||||
? `(SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))`
|
||||
: 'NULL'
|
||||
|
||||
Logger.debug(
|
||||
`[LibraryItemsBookFilters] collapse-series computed includeIds=${bookSeriesToInclude.length} excludeBooks=${booksToExclude.length} ` +
|
||||
`filterGroup=${filterGroup || 'none'} filterValue=${filterValue || 'none'} sortBy=${sortBy}`
|
||||
)
|
||||
if (!includedBookSeriesIds) {
|
||||
Logger.debug(
|
||||
`[LibraryItemsBookFilters] collapse-series produced no include IDs; using library item title fallback ` +
|
||||
`(libraryId=${libraryId}, filterGroup=${filterGroup || 'none'}, filterValue=${filterValue || 'none'}, sortBy=${sortBy})`
|
||||
)
|
||||
}
|
||||
|
||||
if (global.ServerSettings.sortingIgnorePrefix) {
|
||||
bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`titleIgnorePrefix\`)`), 'display_title'])
|
||||
bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByNameIgnorePrefix}, ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title'])
|
||||
} else {
|
||||
bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`title\`)`), 'display_title'])
|
||||
bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByName}, ${fallbackLibraryItemTitle})`), 'display_title'])
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -716,13 +766,14 @@ module.exports = {
|
|||
bookWhere.push(...userPermissionBookWhere.bookWhere)
|
||||
|
||||
let includeAttributes = [[Sequelize.literal('(SELECT max(mp.updatedAt) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id)'), 'recent_progress']]
|
||||
let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = 0 OR mp.isFinished IS NULL)`
|
||||
let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = ${booleanLiteral(false, Database.sequelize)} OR mp.isFinished IS NULL)`
|
||||
|
||||
if (library.settings.onlyShowLaterBooksInContinueSeries) {
|
||||
const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as FLOAT) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = 1 AND mp.userId = :userId AND bs.seriesId = series.id)`
|
||||
const safeSequenceExpr = safeTextToDoubleExpression('bs.sequence', Database.sequelize)
|
||||
const maxSequenceQuery = `(SELECT max(${safeSequenceExpr}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)`
|
||||
includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence'])
|
||||
|
||||
booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as FLOAT) > ${maxSequenceQuery}`
|
||||
booksNotFinishedQuery = booksNotFinishedQuery + ` AND ${safeSequenceExpr} > ${maxSequenceQuery}`
|
||||
}
|
||||
|
||||
const { rows: series, count } = await Database.seriesModel.findAndCountAll({
|
||||
|
|
@ -735,7 +786,7 @@ module.exports = {
|
|||
},
|
||||
// TODO: Simplify queries
|
||||
// Has at least 1 book finished
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = 1)`), {
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)})`), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
// Has at least 1 book not finished (that has a sequence number higher than the highest already read, if library config is toggled)
|
||||
|
|
@ -743,7 +794,7 @@ module.exports = {
|
|||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
// Has no books in progress
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = 0 AND mp.currentTime > 0)`), 0)
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)} AND mp.currentTime > 0)`), 0)
|
||||
],
|
||||
attributes: {
|
||||
include: includeAttributes
|
||||
|
|
@ -757,10 +808,10 @@ module.exports = {
|
|||
attributes: ['bookId', 'sequence'],
|
||||
separate: true,
|
||||
subQuery: false,
|
||||
order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]],
|
||||
order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]],
|
||||
where: {
|
||||
'$book.mediaProgresses.isFinished$': {
|
||||
[Sequelize.Op.or]: [null, 0]
|
||||
[Sequelize.Op.or]: [null, false]
|
||||
}
|
||||
},
|
||||
include: {
|
||||
|
|
@ -854,7 +905,7 @@ module.exports = {
|
|||
{
|
||||
libraryId
|
||||
},
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = 1 OR mp.currentTime > 0))`), 0)
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0))`), 0)
|
||||
],
|
||||
replacements: {
|
||||
userId: user.id,
|
||||
|
|
@ -870,7 +921,7 @@ module.exports = {
|
|||
model: Database.bookModel,
|
||||
where: userPermissionBookWhere.bookWhere
|
||||
},
|
||||
order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]],
|
||||
order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]],
|
||||
limit: 1
|
||||
},
|
||||
subQuery: false,
|
||||
|
|
@ -904,6 +955,7 @@ module.exports = {
|
|||
id: {
|
||||
[Sequelize.Op.in]: booksFromSeriesToInclude
|
||||
}
|
||||
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -1130,7 +1182,7 @@ module.exports = {
|
|||
|
||||
// Search narrators
|
||||
const narratorMatches = []
|
||||
const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
|
||||
const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, ${jsonArrayExpand('b.narrators', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
|
||||
replacements: {
|
||||
libraryId: library.id,
|
||||
limit,
|
||||
|
|
@ -1147,7 +1199,7 @@ module.exports = {
|
|||
|
||||
// Search tags
|
||||
const tagMatches = []
|
||||
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.tags', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
replacements: {
|
||||
libraryId: library.id,
|
||||
limit,
|
||||
|
|
@ -1164,7 +1216,7 @@ module.exports = {
|
|||
|
||||
// Search genres
|
||||
const genreMatches = []
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
replacements: {
|
||||
libraryId: library.id,
|
||||
limit,
|
||||
|
|
@ -1244,7 +1296,7 @@ module.exports = {
|
|||
*/
|
||||
async getGenresWithCount(libraryId) {
|
||||
const genres = []
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
|
||||
replacements: {
|
||||
libraryId
|
||||
},
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ const Database = require('../../Database')
|
|||
const Logger = require('../../Logger')
|
||||
const { profile } = require('../../utils/profiler')
|
||||
const stringifySequelizeQuery = require('../stringifySequelizeQuery')
|
||||
const { jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression, jsonArrayExpand, jsonPathNumber } = require('../sqlDialectHelpers')
|
||||
|
||||
const countCache = new Map()
|
||||
|
||||
|
|
@ -24,10 +25,10 @@ module.exports = {
|
|||
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
|
||||
replacements['userTagsSelected'] = user.permissions.itemTagsSelected
|
||||
if (user.permissions.selectedTagsNotAccessible) {
|
||||
podcastWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0))
|
||||
podcastWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0))
|
||||
} else {
|
||||
podcastWhere.push(
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), {
|
||||
Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
})
|
||||
)
|
||||
|
|
@ -53,7 +54,7 @@ module.exports = {
|
|||
const replacements = {}
|
||||
|
||||
if (['genres', 'tags'].includes(group)) {
|
||||
mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), {
|
||||
mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
})
|
||||
replacements.filterValue = value
|
||||
|
|
@ -87,12 +88,12 @@ module.exports = {
|
|||
return [[Sequelize.literal('libraryItem.mtime'), dir]]
|
||||
} else if (sortBy === 'media.metadata.author') {
|
||||
const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'
|
||||
return [[Sequelize.literal(`\`podcast\`.\`author\` COLLATE NOCASE ${nullDir}`)]]
|
||||
return [[Sequelize.literal(`${noCaseSortExpression('podcast.author', Database.sequelize)} ${nullDir}`)]]
|
||||
} else if (sortBy === 'media.metadata.title') {
|
||||
if (global.ServerSettings.sortingIgnorePrefix) {
|
||||
return [[Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]]
|
||||
return [[Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]]
|
||||
} else {
|
||||
return [[Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]]
|
||||
return [[Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]]
|
||||
}
|
||||
} else if (sortBy === 'media.numTracks') {
|
||||
return [['numEpisodes', dir]]
|
||||
|
|
@ -455,7 +456,7 @@ module.exports = {
|
|||
|
||||
// Search tags
|
||||
const tagMatches = []
|
||||
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.tags', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
replacements: {
|
||||
libraryId: library.id,
|
||||
limit,
|
||||
|
|
@ -472,7 +473,7 @@ module.exports = {
|
|||
|
||||
// Search genres
|
||||
const genreMatches = []
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
replacements: {
|
||||
libraryId: library.id,
|
||||
limit,
|
||||
|
|
@ -563,12 +564,12 @@ module.exports = {
|
|||
* @returns {Promise<{ totalSize:number, totalDuration:number, numAudioFiles:number, totalItems:number}>}
|
||||
*/
|
||||
async getPodcastLibraryStats(libraryId) {
|
||||
const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = "podcast" AND li.libraryId = :libraryId;`, {
|
||||
const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = 'podcast' AND li.libraryId = :libraryId;`, {
|
||||
replacements: {
|
||||
libraryId
|
||||
}
|
||||
})
|
||||
const [statResults] = await Database.sequelize.query(`SELECT SUM(json_extract(pe.audioFile, '$.duration')) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, {
|
||||
const [statResults] = await Database.sequelize.query(`SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, {
|
||||
replacements: {
|
||||
libraryId
|
||||
}
|
||||
|
|
@ -588,7 +589,7 @@ module.exports = {
|
|||
*/
|
||||
async getGenresWithCount(libraryId) {
|
||||
const genres = []
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
|
||||
replacements: {
|
||||
libraryId
|
||||
},
|
||||
|
|
@ -611,7 +612,7 @@ module.exports = {
|
|||
*/
|
||||
async getLongestPodcasts(libraryId, limit) {
|
||||
const podcasts = await Database.podcastModel.findAll({
|
||||
attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(json_extract(pe.audioFile, '$.duration')) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']],
|
||||
attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']],
|
||||
include: {
|
||||
model: Database.libraryItemModel,
|
||||
attributes: ['id', 'libraryId'],
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ const Sequelize = require('sequelize')
|
|||
const Logger = require('../../Logger')
|
||||
const Database = require('../../Database')
|
||||
const libraryItemsBookFilters = require('./libraryItemsBookFilters')
|
||||
const { booleanLiteral, jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression } = require('../sqlDialectHelpers')
|
||||
|
||||
module.exports = {
|
||||
decode(text) {
|
||||
|
|
@ -60,7 +61,7 @@ module.exports = {
|
|||
// TODO: Simplify and break-out
|
||||
let attrQuery = null
|
||||
if (['genres', 'tags', 'narrators'].includes(filterGroup)) {
|
||||
attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (SELECT count(*) FROM json_each(b.${filterGroup}) WHERE json_valid(b.${filterGroup}) AND json_each.value = :filterValue) > 0`
|
||||
attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ${jsonArrayContainsValue(`b.${filterGroup}`, 'filterValue', Database.sequelize)} > 0`
|
||||
userPermissionBookWhere.replacements.filterValue = filterValue
|
||||
} else if (filterGroup === 'authors') {
|
||||
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs, bookAuthors ba WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ba.bookId = b.id AND ba.authorId = :filterValue'
|
||||
|
|
@ -73,18 +74,18 @@ module.exports = {
|
|||
userPermissionBookWhere.replacements.filterValue = filterValue
|
||||
} else if (filterGroup === 'progress') {
|
||||
if (filterValue === 'not-finished') {
|
||||
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)'
|
||||
attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})`
|
||||
userPermissionBookWhere.replacements.userId = user.id
|
||||
} else if (filterValue === 'finished') {
|
||||
const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)'
|
||||
const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})`
|
||||
seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0))
|
||||
userPermissionBookWhere.replacements.userId = user.id
|
||||
} else if (filterValue === 'not-started') {
|
||||
const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = 1 OR mp.currentTime > 0)'
|
||||
const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0)`
|
||||
seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0))
|
||||
userPermissionBookWhere.replacements.userId = user.id
|
||||
} else if (filterValue === 'in-progress') {
|
||||
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = 0'
|
||||
attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)}`
|
||||
userPermissionBookWhere.replacements.userId = user.id
|
||||
}
|
||||
}
|
||||
|
|
@ -95,13 +96,13 @@ module.exports = {
|
|||
if (!attrQuery) attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id'
|
||||
|
||||
if (!user.canAccessExplicitContent) {
|
||||
attrQuery += ' AND b.explicit = 0'
|
||||
attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}`
|
||||
}
|
||||
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
|
||||
if (user.permissions.selectedTagsNotAccessible) {
|
||||
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0'
|
||||
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0`
|
||||
} else {
|
||||
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0'
|
||||
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -128,9 +129,9 @@ module.exports = {
|
|||
order.push(['createdAt', dir])
|
||||
} else if (sortBy === 'name') {
|
||||
if (global.ServerSettings.sortingIgnorePrefix) {
|
||||
order.push([Sequelize.literal('nameIgnorePrefix COLLATE NOCASE'), dir])
|
||||
order.push([Sequelize.literal(noCaseSortExpression('nameIgnorePrefix', Database.sequelize)), dir])
|
||||
} else {
|
||||
order.push([Sequelize.literal('`series`.`name` COLLATE NOCASE'), dir])
|
||||
order.push([Sequelize.literal(noCaseSortExpression('series.name', Database.sequelize)), dir])
|
||||
}
|
||||
} else if (sortBy === 'totalDuration') {
|
||||
seriesAttributes.include.push([Sequelize.literal('(SELECT SUM(b.duration) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND b.id = bs.bookId)'), 'totalDuration'])
|
||||
|
|
|
|||
91
server/utils/sqlDialectHelpers.js
Normal file
91
server/utils/sqlDialectHelpers.js
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
function getDialect(sequelize) {
|
||||
if (!sequelize) return 'sqlite'
|
||||
if (typeof sequelize.getDialect === 'function') return sequelize.getDialect()
|
||||
return sequelize.dialect?.name || 'sqlite'
|
||||
}
|
||||
|
||||
function isPostgres(sequelize) {
|
||||
return getDialect(sequelize) === 'postgres'
|
||||
}
|
||||
|
||||
function booleanLiteral(value, sequelize) {
|
||||
if (isPostgres(sequelize)) return value ? 'TRUE' : 'FALSE'
|
||||
return value ? '1' : '0'
|
||||
}
|
||||
|
||||
function noCaseSortExpression(columnExpression, sequelize) {
|
||||
if (isPostgres(sequelize)) return `LOWER(${columnExpression})`
|
||||
return `${columnExpression} COLLATE NOCASE`
|
||||
}
|
||||
|
||||
function coalesceFunctionName(sequelize) {
|
||||
return isPostgres(sequelize) ? 'COALESCE' : 'IFNULL'
|
||||
}
|
||||
|
||||
function jsonArrayContainsAny(columnExpression, bindName, sequelize) {
|
||||
if (isPostgres(sequelize)) {
|
||||
return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:${bindName}))`
|
||||
}
|
||||
return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value IN (:${bindName}))`
|
||||
}
|
||||
|
||||
function jsonArrayContainsValue(columnExpression, bindName, sequelize) {
|
||||
if (isPostgres(sequelize)) {
|
||||
return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :${bindName})`
|
||||
}
|
||||
return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value = :${bindName})`
|
||||
}
|
||||
|
||||
function jsonArrayExpand(columnExpression, sequelize, options = {}) {
|
||||
const alias = options.alias || 'json_each'
|
||||
const textValues = options.textValues !== false
|
||||
if (isPostgres(sequelize)) {
|
||||
const fn = textValues ? 'jsonb_array_elements_text' : 'jsonb_array_elements'
|
||||
return `${fn}(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS ${alias}(value)`
|
||||
}
|
||||
return `json_each(${columnExpression})`
|
||||
}
|
||||
|
||||
function jsonPathText(columnExpression, pathSegments, sequelize) {
|
||||
const path = Array.isArray(pathSegments) ? pathSegments : [pathSegments]
|
||||
if (isPostgres(sequelize)) {
|
||||
return `${columnExpression}::jsonb #>> '{${path.join(',')}}'`
|
||||
}
|
||||
return `json_extract(${columnExpression}, '$.${path.join('.')}')`
|
||||
}
|
||||
|
||||
function jsonPathNumber(columnExpression, pathSegments, sequelize) {
|
||||
if (isPostgres(sequelize)) {
|
||||
return `NULLIF(${jsonPathText(columnExpression, pathSegments, sequelize)}, '')::double precision`
|
||||
}
|
||||
return `json_extract(${columnExpression}, '$.${[].concat(pathSegments).join('.')}')`
|
||||
}
|
||||
|
||||
function safeTextToDoubleExpression(columnExpression, sequelize) {
|
||||
if (isPostgres(sequelize)) {
|
||||
return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(${columnExpression})::double precision ELSE NULL END`
|
||||
}
|
||||
return `CAST(${columnExpression} AS FLOAT)`
|
||||
}
|
||||
|
||||
function safeTextToIntegerExpression(columnExpression, sequelize) {
|
||||
if (isPostgres(sequelize)) {
|
||||
return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?\\d+$' THEN BTRIM(${columnExpression})::integer ELSE NULL END`
|
||||
}
|
||||
return `CAST(${columnExpression} AS INTEGER)`
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getDialect,
|
||||
isPostgres,
|
||||
booleanLiteral,
|
||||
noCaseSortExpression,
|
||||
coalesceFunctionName,
|
||||
jsonArrayContainsAny,
|
||||
jsonArrayContainsValue,
|
||||
jsonArrayExpand,
|
||||
jsonPathText,
|
||||
jsonPathNumber,
|
||||
safeTextToDoubleExpression,
|
||||
safeTextToIntegerExpression
|
||||
}
|
||||
101
test/server/Database.test.js
Normal file
101
test/server/Database.test.js
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
const { expect } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
|
||||
const fs = require('../../server/libs/fsExtra')
|
||||
const Database = require('../../server/Database')
|
||||
|
||||
describe('Database', () => {
|
||||
let originalDialect
|
||||
let originalSequelize
|
||||
let originalEnv
|
||||
|
||||
beforeEach(() => {
|
||||
originalDialect = Database.dialect
|
||||
originalSequelize = Database.sequelize
|
||||
originalEnv = {
|
||||
DB_DIALECT: process.env.DB_DIALECT,
|
||||
DATABASE_URL: process.env.DATABASE_URL
|
||||
}
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
Database.dialect = originalDialect
|
||||
Database.sequelize = originalSequelize
|
||||
|
||||
if (originalEnv.DB_DIALECT === undefined) delete process.env.DB_DIALECT
|
||||
else process.env.DB_DIALECT = originalEnv.DB_DIALECT
|
||||
|
||||
if (originalEnv.DATABASE_URL === undefined) delete process.env.DATABASE_URL
|
||||
else process.env.DATABASE_URL = originalEnv.DATABASE_URL
|
||||
|
||||
sinon.restore()
|
||||
})
|
||||
|
||||
describe('getConfiguredDialect', () => {
|
||||
it('should default to sqlite when no env variables are set', () => {
|
||||
delete process.env.DB_DIALECT
|
||||
delete process.env.DATABASE_URL
|
||||
|
||||
expect(Database.getConfiguredDialect()).to.equal('sqlite')
|
||||
})
|
||||
|
||||
it('should use explicit DB_DIALECT value when valid', () => {
|
||||
process.env.DB_DIALECT = 'postgres'
|
||||
process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite'
|
||||
|
||||
expect(Database.getConfiguredDialect()).to.equal('postgres')
|
||||
})
|
||||
|
||||
it('should infer postgres dialect from DATABASE_URL', () => {
|
||||
delete process.env.DB_DIALECT
|
||||
process.env.DATABASE_URL = 'postgresql://user:pass@localhost:5432/abs'
|
||||
|
||||
expect(Database.getConfiguredDialect()).to.equal('postgres')
|
||||
})
|
||||
|
||||
it('should fallback to sqlite for unsupported DB_DIALECT', () => {
|
||||
process.env.DB_DIALECT = 'mysql'
|
||||
process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite'
|
||||
|
||||
expect(Database.getConfiguredDialect()).to.equal('sqlite')
|
||||
})
|
||||
})
|
||||
|
||||
describe('checkHasDb', () => {
|
||||
it('should not check sqlite file existence in postgres mode', async () => {
|
||||
Database.dialect = 'postgres'
|
||||
const pathExistsStub = sinon.stub(fs, 'pathExists')
|
||||
|
||||
const hasDb = await Database.checkHasDb()
|
||||
|
||||
expect(hasDb).to.equal(true)
|
||||
expect(pathExistsStub.called).to.equal(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('checkHasTables', () => {
|
||||
it('should return true when at least one table exists', async () => {
|
||||
Database.sequelize = {
|
||||
getQueryInterface: () => ({
|
||||
showAllTables: sinon.stub().resolves(['users'])
|
||||
})
|
||||
}
|
||||
|
||||
const hasTables = await Database.checkHasTables()
|
||||
|
||||
expect(hasTables).to.equal(true)
|
||||
})
|
||||
|
||||
it('should return false when no tables exist', async () => {
|
||||
Database.sequelize = {
|
||||
getQueryInterface: () => ({
|
||||
showAllTables: sinon.stub().resolves([])
|
||||
})
|
||||
}
|
||||
|
||||
const hasTables = await Database.checkHasTables()
|
||||
|
||||
expect(hasTables).to.equal(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -189,6 +189,52 @@ describe('MigrationManager', () => {
|
|||
expect(loggerInfoStub.calledWith(sinon.match('Restored the original database'))).to.be.true
|
||||
expect(processExitStub.calledOnce).to.be.true
|
||||
})
|
||||
|
||||
it('should skip sqlite backup workflow for postgres migrations', async () => {
|
||||
// Arrange
|
||||
migrationManager.serverVersion = '1.2.0'
|
||||
migrationManager.databaseVersion = '1.1.0'
|
||||
migrationManager.maxVersion = '1.1.0'
|
||||
migrationManager.initialized = true
|
||||
sequelizeStub.getDialect.returns('postgres')
|
||||
|
||||
umzugStub.migrations.resolves([{ name: 'v1.2.0-migration.js' }])
|
||||
umzugStub.executed.resolves([{ name: 'v1.1.0-migration.js' }])
|
||||
|
||||
// Act
|
||||
await migrationManager.runMigrations()
|
||||
|
||||
// Assert
|
||||
expect(umzugStub.up.calledOnce).to.be.true
|
||||
expect(fsCopyStub.called).to.be.false
|
||||
expect(fsRemoveStub.called).to.be.false
|
||||
})
|
||||
})
|
||||
|
||||
describe('tableExists', () => {
|
||||
it('should use queryInterface.tableExists when available', async () => {
|
||||
const tableExistsStub = sinon.stub().resolves(true)
|
||||
const sequelize = sinon.createStubInstance(Sequelize)
|
||||
sequelize.getQueryInterface.returns({ tableExists: tableExistsStub })
|
||||
const manager = new MigrationManager(sequelize, false, configPath)
|
||||
|
||||
const exists = await manager.tableExists('migrationsMeta')
|
||||
|
||||
expect(exists).to.equal(true)
|
||||
expect(tableExistsStub.calledOnceWithExactly('migrationsMeta')).to.equal(true)
|
||||
})
|
||||
|
||||
it('should fallback to showAllTables and support object table names', async () => {
|
||||
const sequelize = sinon.createStubInstance(Sequelize)
|
||||
sequelize.getQueryInterface.returns({
|
||||
showAllTables: sinon.stub().resolves([{ tableName: 'migrationsMeta' }])
|
||||
})
|
||||
const manager = new MigrationManager(sequelize, false, configPath)
|
||||
|
||||
const exists = await manager.tableExists('migrationsMeta')
|
||||
|
||||
expect(exists).to.equal(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('fetchVersionsFromDatabase', () => {
|
||||
|
|
@ -280,6 +326,20 @@ describe('MigrationManager', () => {
|
|||
expect(error.message).to.equal('Database query failed')
|
||||
}
|
||||
})
|
||||
|
||||
it('should support lowercase maxversion alias from postgres drivers', async () => {
|
||||
const sequelize = sinon.createStubInstance(Sequelize)
|
||||
sequelize.query.onFirstCall().resolves([{ version: '1.1.0' }])
|
||||
sequelize.query.onSecondCall().resolves([{ maxversion: '1.2.0' }])
|
||||
|
||||
const manager = new MigrationManager(sequelize, false, configPath)
|
||||
manager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves()
|
||||
|
||||
await manager.fetchVersionsFromDatabase()
|
||||
|
||||
expect(manager.databaseVersion).to.equal('1.1.0')
|
||||
expect(manager.maxVersion).to.equal('1.2.0')
|
||||
})
|
||||
})
|
||||
|
||||
describe('updateMaxVersion', () => {
|
||||
|
|
|
|||
|
|
@ -88,6 +88,23 @@ describe('migration-v2.15.0-series-column-unique', () => {
|
|||
await queryInterface.dropTable('Series')
|
||||
await queryInterface.dropTable('BookSeries')
|
||||
})
|
||||
|
||||
it('should skip NOCASE reindex on postgres dialect', async () => {
|
||||
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
|
||||
const originalQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize)
|
||||
const queryStub = sinon.stub(queryInterface.sequelize, 'query').callsFake((sql, options) => {
|
||||
if (sql === 'REINDEX NOCASE;') {
|
||||
throw new Error('Unexpected NOCASE reindex in postgres mode')
|
||||
}
|
||||
return originalQuery(sql, options)
|
||||
})
|
||||
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
expect(queryStub.neverCalledWith('REINDEX NOCASE;')).to.be.true
|
||||
expect(loggerInfoStub.calledWith(sinon.match('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.be.true
|
||||
})
|
||||
|
||||
it('upgrade with no duplicate series', async () => {
|
||||
// Add some entries to the Series table using the UUID for the ids
|
||||
await queryInterface.bulkInsert('Series', [
|
||||
|
|
|
|||
41
test/server/migrations/v2.15.1-reindex-nocase.test.js
Normal file
41
test/server/migrations/v2.15.1-reindex-nocase.test.js
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
const { expect } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
|
||||
const { up, down } = require('../../../server/migrations/v2.15.1-reindex-nocase')
|
||||
const Logger = require('../../../server/Logger')
|
||||
|
||||
describe('migration-v2.15.1-reindex-nocase', () => {
|
||||
afterEach(() => {
|
||||
sinon.restore()
|
||||
})
|
||||
|
||||
it('should skip reindex on non-sqlite dialect', async () => {
|
||||
const queryStub = sinon.stub().resolves()
|
||||
const loggerInfoStub = sinon.stub(Logger, 'info')
|
||||
const queryInterface = {
|
||||
sequelize: {
|
||||
getDialect: () => 'postgres',
|
||||
query: queryStub
|
||||
}
|
||||
}
|
||||
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
expect(queryStub.called).to.equal(false)
|
||||
expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.equal(true)
|
||||
})
|
||||
|
||||
it('should log no-op on down migration', async () => {
|
||||
const loggerInfoStub = sinon.stub(Logger, 'info')
|
||||
const queryInterface = {
|
||||
sequelize: {
|
||||
getDialect: () => 'postgres',
|
||||
query: sinon.stub().resolves()
|
||||
}
|
||||
}
|
||||
|
||||
await down({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] No action required for downgrade'))).to.equal(true)
|
||||
})
|
||||
})
|
||||
|
|
@ -34,6 +34,16 @@ describe('migration-v2.17.3-fk-constraints', () => {
|
|||
await queryInterface.dropAllTables()
|
||||
})
|
||||
|
||||
it('should skip sqlite-specific migration on non-sqlite dialect', async () => {
|
||||
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
|
||||
const queryStub = sinon.stub(queryInterface.sequelize, 'query').resolves([])
|
||||
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
expect(queryStub.called).to.equal(false)
|
||||
expect(loggerInfoStub.calledWith(sinon.match('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect'))).to.equal(true)
|
||||
})
|
||||
|
||||
it('should fix table foreign key constraints', async () => {
|
||||
// Create tables with missing foreign key constraints: libraryItems, feeds, mediaItemShares, playbackSessions, playlistMediaItems, mediaProgresses
|
||||
await queryInterface.sequelize.query('CREATE TABLE `libraryItems` (`id` UUID UNIQUE PRIMARY KEY, `libraryId` UUID REFERENCES `libraries` (`id`), `libraryFolderId` UUID REFERENCES `libraryFolders` (`id`));')
|
||||
|
|
|
|||
|
|
@ -47,6 +47,18 @@ describe('Migration v2.19.1-copy-title-to-library-items', () => {
|
|||
})
|
||||
|
||||
describe('up', () => {
|
||||
it('should skip sqlite-specific migration on non-sqlite dialect', async () => {
|
||||
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
|
||||
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const table = await queryInterface.describeTable('libraryItems')
|
||||
expect(table).to.have.property('id')
|
||||
expect(table).to.not.have.property('title')
|
||||
expect(table).to.not.have.property('titleIgnorePrefix')
|
||||
expect(loggerInfoStub.calledWith(sinon.match('[2.19.1 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true
|
||||
})
|
||||
|
||||
it('should copy title and titleIgnorePrefix to libraryItems', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
|
|
|
|||
|
|
@ -75,6 +75,18 @@ describe('Migration v2.19.4-improve-podcast-queries', () => {
|
|||
})
|
||||
|
||||
describe('up', () => {
|
||||
it('should skip sqlite-specific migration on non-sqlite dialect', async () => {
|
||||
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
|
||||
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const podcastsTable = await queryInterface.describeTable('podcasts')
|
||||
const mediaProgressesTable = await queryInterface.describeTable('mediaProgresses')
|
||||
expect(podcastsTable).to.not.have.property('numEpisodes')
|
||||
expect(mediaProgressesTable).to.not.have.property('podcastId')
|
||||
expect(loggerInfoStub.calledWith(sinon.match('[2.19.4 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true
|
||||
})
|
||||
|
||||
it('should add numEpisodes column to podcasts', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
|
|
|
|||
70
test/server/models/MediaProgress.test.js
Normal file
70
test/server/models/MediaProgress.test.js
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
const { expect } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
|
||||
const Logger = require('../../../server/Logger')
|
||||
const MediaProgress = require('../../../server/models/MediaProgress')
|
||||
|
||||
function createProgressInstance() {
|
||||
const progress = Object.create(MediaProgress.prototype)
|
||||
|
||||
Object.defineProperties(progress, {
|
||||
id: { value: 'progress-1', writable: true, configurable: true },
|
||||
mediaItemId: { value: 'media-1', writable: true, configurable: true },
|
||||
duration: { value: 3600, writable: true, configurable: true },
|
||||
currentTime: { value: 120, writable: true, configurable: true },
|
||||
isFinished: { value: false, writable: true, configurable: true },
|
||||
hideFromContinueListening: { value: false, writable: true, configurable: true },
|
||||
extraData: { value: {}, writable: true, configurable: true }
|
||||
})
|
||||
|
||||
progress.changed = sinon.stub().returns(false)
|
||||
progress.set = sinon.stub().callsFake((payload) => Object.assign(progress, payload))
|
||||
progress.save = sinon.stub().resolves()
|
||||
progress.reload = sinon.stub().resolves()
|
||||
progress.constructor = {
|
||||
update: sinon.stub().resolves(),
|
||||
sequelize: {
|
||||
escape: (value) => `'${value.toISOString ? value.toISOString() : value}'`
|
||||
}
|
||||
}
|
||||
|
||||
return progress
|
||||
}
|
||||
|
||||
describe('MediaProgress', () => {
|
||||
afterEach(() => {
|
||||
sinon.restore()
|
||||
})
|
||||
|
||||
describe('applyProgressUpdate', () => {
|
||||
it('should update updatedAt via model update for valid lastUpdate', async () => {
|
||||
const progress = createProgressInstance()
|
||||
const infoStub = sinon.stub(Logger, 'info')
|
||||
const lastUpdate = '2026-03-03T01:00:00.000Z'
|
||||
|
||||
await progress.applyProgressUpdate({ currentTime: 130, lastUpdate })
|
||||
|
||||
expect(progress.save.calledOnce).to.equal(true)
|
||||
expect(progress.constructor.update.calledOnce).to.equal(true)
|
||||
expect(progress.constructor.update.firstCall.args[0].updatedAt.toISOString()).to.equal(new Date(lastUpdate).toISOString())
|
||||
expect(progress.constructor.update.firstCall.args[1]).to.deep.equal({
|
||||
where: { id: 'progress-1' },
|
||||
silent: true
|
||||
})
|
||||
expect(progress.reload.calledOnce).to.equal(true)
|
||||
expect(infoStub.calledWithMatch('[MediaProgress] Manually setting updatedAt')).to.equal(true)
|
||||
})
|
||||
|
||||
it('should skip manual updatedAt update when lastUpdate is invalid', async () => {
|
||||
const progress = createProgressInstance()
|
||||
const warnStub = sinon.stub(Logger, 'warn')
|
||||
|
||||
await progress.applyProgressUpdate({ currentTime: 130, lastUpdate: 'invalid-date' })
|
||||
|
||||
expect(progress.save.calledOnce).to.equal(true)
|
||||
expect(progress.constructor.update.called).to.equal(false)
|
||||
expect(progress.reload.called).to.equal(false)
|
||||
expect(warnStub.calledWithMatch('[MediaProgress] Invalid date provided for lastUpdate')).to.equal(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
96
test/server/models/User.test.js
Normal file
96
test/server/models/User.test.js
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
const { expect } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
const User = require('../../../server/models/User')
|
||||
|
||||
describe('User model', () => {
|
||||
describe('getUserByIdOrOldId', () => {
|
||||
let originalSequelize
|
||||
|
||||
beforeEach(() => {
|
||||
originalSequelize = User.sequelize
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
User.sequelize = originalSequelize
|
||||
sinon.restore()
|
||||
})
|
||||
|
||||
it('should resolve UUID ids via primary-key lookup on postgres', async () => {
|
||||
User.sequelize = {
|
||||
getDialect: () => 'postgres',
|
||||
models: {
|
||||
mediaProgress: {}
|
||||
}
|
||||
}
|
||||
|
||||
const user = { id: 'e8e677b2-da16-4220-ab67-443b7714caf9' }
|
||||
const findByPkStub = sinon.stub(User, 'findByPk').resolves(user)
|
||||
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
|
||||
|
||||
const result = await User.getUserByIdOrOldId('e8e677b2-da16-4220-ab67-443b7714caf9')
|
||||
|
||||
expect(result).to.equal(user)
|
||||
expect(findByPkStub.calledOnce).to.equal(true)
|
||||
expect(findOneStub.called).to.equal(false)
|
||||
})
|
||||
|
||||
it('should resolve uppercase UUID ids via primary-key lookup on postgres', async () => {
|
||||
User.sequelize = {
|
||||
getDialect: () => 'postgres',
|
||||
models: {
|
||||
mediaProgress: {}
|
||||
}
|
||||
}
|
||||
|
||||
const uppercaseUuid = 'E8E677B2-DA16-4220-AB67-443B7714CAF9'
|
||||
const user = { id: uppercaseUuid }
|
||||
const findByPkStub = sinon.stub(User, 'findByPk').resolves(user)
|
||||
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
|
||||
|
||||
const result = await User.getUserByIdOrOldId(uppercaseUuid)
|
||||
|
||||
expect(result).to.equal(user)
|
||||
expect(findByPkStub.calledOnceWithExactly(uppercaseUuid, { include: User.sequelize.models.mediaProgress })).to.equal(true)
|
||||
expect(findOneStub.called).to.equal(false)
|
||||
})
|
||||
|
||||
it('should query legacy oldUserId with postgres-safe JSON matcher', async () => {
|
||||
User.sequelize = {
|
||||
getDialect: () => 'postgres',
|
||||
models: {
|
||||
mediaProgress: {}
|
||||
}
|
||||
}
|
||||
|
||||
const findByPkStub = sinon.stub(User, 'findByPk').resolves(null)
|
||||
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
|
||||
|
||||
await User.getUserByIdOrOldId('root')
|
||||
|
||||
expect(findByPkStub.called).to.equal(false)
|
||||
expect(findOneStub.calledOnce).to.equal(true)
|
||||
|
||||
const options = findOneStub.firstCall.args[0]
|
||||
expect(options.where.attribute.val).to.equal("extradata#>>'{oldUserId}'")
|
||||
expect(options.where.logic).to.equal('root')
|
||||
})
|
||||
|
||||
it('should keep sqlite oldUserId matcher unchanged', async () => {
|
||||
User.sequelize = {
|
||||
getDialect: () => 'sqlite',
|
||||
models: {
|
||||
mediaProgress: {}
|
||||
}
|
||||
}
|
||||
|
||||
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
|
||||
|
||||
await User.getUserByIdOrOldId('root')
|
||||
|
||||
expect(findOneStub.calledOnce).to.equal(true)
|
||||
|
||||
const options = findOneStub.firstCall.args[0]
|
||||
expect(options.where).to.deep.equal({ 'extraData.oldUserId': 'root' })
|
||||
})
|
||||
})
|
||||
})
|
||||
146
test/server/scripts/migrateSqliteToPostgres.test.js
Normal file
146
test/server/scripts/migrateSqliteToPostgres.test.js
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
const { expect } = require('chai')
|
||||
const sqlite3 = require('sqlite3')
|
||||
|
||||
const {
|
||||
normalizeJson,
|
||||
isIntegerCompatible,
|
||||
convertValue,
|
||||
findOverlongVarcharValues,
|
||||
findIntegerTypeIssues
|
||||
} = require('../../../server/scripts/migrateSqliteToPostgres')
|
||||
|
||||
function openMemoryDb() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const db = new sqlite3.Database(':memory:', (error) => {
|
||||
if (error) return reject(error)
|
||||
resolve(db)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function run(db, sql, params = []) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.run(sql, params, (error) => {
|
||||
if (error) return reject(error)
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function close(db) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.close((error) => {
|
||||
if (error) return reject(error)
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
describe('migrateSqliteToPostgres script helpers', () => {
|
||||
it('should keep malformed JSON payloads insertable by returning valid JSON text', () => {
|
||||
const malformed = '"{"x",1}"'
|
||||
const normalized = normalizeJson(malformed)
|
||||
|
||||
expect(() => JSON.parse(normalized)).to.not.throw()
|
||||
expect(JSON.parse(normalized)).to.equal('"{"x",1}"')
|
||||
})
|
||||
|
||||
it('should detect overlong varchar values before migration', async () => {
|
||||
const db = await openMemoryDb()
|
||||
|
||||
try {
|
||||
await run(db, 'CREATE TABLE books (subtitle TEXT)')
|
||||
await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['x'.repeat(300)])
|
||||
await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['ok'])
|
||||
|
||||
const tablesToMigrate = [{ sqliteTable: 'books', postgresTable: 'books' }]
|
||||
const pgColumnsByTable = new Map([
|
||||
[
|
||||
'books',
|
||||
new Map([
|
||||
[
|
||||
'subtitle',
|
||||
{
|
||||
column_name: 'subtitle',
|
||||
data_type: 'character varying',
|
||||
character_maximum_length: 255
|
||||
}
|
||||
]
|
||||
])
|
||||
]
|
||||
])
|
||||
|
||||
const issues = await findOverlongVarcharValues(db, tablesToMigrate, pgColumnsByTable)
|
||||
|
||||
expect(issues).to.have.length(1)
|
||||
expect(issues[0]).to.include({
|
||||
sqliteTable: 'books',
|
||||
sqliteColumn: 'subtitle',
|
||||
postgresTable: 'books',
|
||||
postgresColumn: 'subtitle',
|
||||
maxLength: 255,
|
||||
actualMaxLength: 300,
|
||||
overCount: 1
|
||||
})
|
||||
} finally {
|
||||
await close(db)
|
||||
}
|
||||
})
|
||||
|
||||
it('should detect non-integer values for integer target columns', async () => {
|
||||
const db = await openMemoryDb()
|
||||
|
||||
try {
|
||||
await run(db, 'CREATE TABLE playbackSessions (timeListening REAL)')
|
||||
await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [25.802536999999997])
|
||||
await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [42])
|
||||
|
||||
const tablesToMigrate = [{ sqliteTable: 'playbackSessions', postgresTable: 'playbackSessions' }]
|
||||
const pgColumnsByTable = new Map([
|
||||
[
|
||||
'playbackSessions',
|
||||
new Map([
|
||||
[
|
||||
'timelistening',
|
||||
{
|
||||
column_name: 'timeListening',
|
||||
data_type: 'integer'
|
||||
}
|
||||
]
|
||||
])
|
||||
]
|
||||
])
|
||||
|
||||
const issues = await findIntegerTypeIssues(db, tablesToMigrate, pgColumnsByTable)
|
||||
|
||||
expect(issues).to.have.length(1)
|
||||
expect(issues[0].sqliteTable).to.equal('playbackSessions')
|
||||
expect(issues[0].sqliteColumn).to.equal('timeListening')
|
||||
expect(issues[0].postgresType).to.equal('integer')
|
||||
expect(issues[0].badCount).to.equal(1)
|
||||
} finally {
|
||||
await close(db)
|
||||
}
|
||||
})
|
||||
|
||||
it('should only accept integer-compatible values for integer columns', () => {
|
||||
expect(isIntegerCompatible(10)).to.equal(true)
|
||||
expect(isIntegerCompatible('10')).to.equal(true)
|
||||
expect(isIntegerCompatible(10.5)).to.equal(false)
|
||||
expect(isIntegerCompatible('10.5')).to.equal(false)
|
||||
})
|
||||
|
||||
it('should coerce integer-like strings for postgres integer columns', () => {
|
||||
expect(convertValue('42', { data_type: 'integer' })).to.equal(42)
|
||||
expect(convertValue('-7', { data_type: 'bigint' })).to.equal(-7)
|
||||
expect(convertValue('4.2', { data_type: 'integer' })).to.equal('4.2')
|
||||
})
|
||||
|
||||
it('should always return valid json text for postgres json columns', () => {
|
||||
const convertedObject = convertValue({ a: 1 }, { data_type: 'jsonb', udt_name: 'jsonb' })
|
||||
const convertedMalformed = convertValue('"{"x",1}"', { data_type: 'jsonb', udt_name: 'jsonb' })
|
||||
|
||||
expect(JSON.parse(convertedObject)).to.deep.equal({ a: 1 })
|
||||
expect(JSON.parse(convertedMalformed)).to.equal('"{"x",1}"')
|
||||
})
|
||||
})
|
||||
133
test/server/utils/queries/libraryItemsBookFilters.test.js
Normal file
133
test/server/utils/queries/libraryItemsBookFilters.test.js
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
const { expect } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
|
||||
const Database = require('../../../../server/Database')
|
||||
const Logger = require('../../../../server/Logger')
|
||||
const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters')
|
||||
|
||||
function createSequelizeStub(dialect, models) {
|
||||
return {
|
||||
getDialect: () => dialect,
|
||||
escape: (value) => `'${String(value).replace(/'/g, "''")}'`,
|
||||
random: () => ({ fn: 'random' }),
|
||||
models
|
||||
}
|
||||
}
|
||||
|
||||
function createModelStubs() {
|
||||
return {
|
||||
book: {
|
||||
findAndCountAll: sinon.stub(),
|
||||
findAll: sinon.stub(),
|
||||
count: sinon.stub()
|
||||
},
|
||||
libraryItem: {},
|
||||
feed: {},
|
||||
bookSeries: {},
|
||||
series: {
|
||||
findAll: sinon.stub()
|
||||
},
|
||||
bookAuthor: {},
|
||||
author: {},
|
||||
mediaProgress: {},
|
||||
mediaItemShare: {}
|
||||
}
|
||||
}
|
||||
|
||||
describe('libraryItemsBookFilters postgres query safety', () => {
|
||||
let originalSequelize
|
||||
let originalServerSettings
|
||||
let modelStubs
|
||||
|
||||
beforeEach(() => {
|
||||
originalSequelize = Database.sequelize
|
||||
originalServerSettings = global.ServerSettings
|
||||
modelStubs = createModelStubs()
|
||||
global.ServerSettings = {
|
||||
sortingIgnorePrefix: false
|
||||
}
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
Database.sequelize = originalSequelize
|
||||
global.ServerSettings = originalServerSettings
|
||||
sinon.restore()
|
||||
})
|
||||
|
||||
it('should avoid generating IN () when collapse-series has no include ids', async () => {
|
||||
Database.sequelize = createSequelizeStub('postgres', modelStubs)
|
||||
modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 })
|
||||
|
||||
sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({
|
||||
booksToExclude: [],
|
||||
bookSeriesToInclude: []
|
||||
})
|
||||
const debugStub = sinon.stub(Logger, 'debug')
|
||||
|
||||
await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, true, [], 20, 0)
|
||||
|
||||
const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0]
|
||||
const displayTitleExpression = findOptions.attributes.include[0][0].val
|
||||
|
||||
expect(displayTitleExpression).to.include('COALESCE(NULL, libraryItem.title)')
|
||||
expect(displayTitleExpression).to.not.include('IN ()')
|
||||
expect(debugStub.calledWithMatch('collapse-series produced no include IDs')).to.equal(true)
|
||||
})
|
||||
|
||||
it('should escape collapse-series ids safely for postgres subquery', async () => {
|
||||
Database.sequelize = createSequelizeStub('postgres', modelStubs)
|
||||
modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 })
|
||||
|
||||
sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({
|
||||
booksToExclude: [],
|
||||
bookSeriesToInclude: [{ id: "series-id-'quoted'", numBooks: 2, libraryItemIds: [] }]
|
||||
})
|
||||
global.ServerSettings.sortingIgnorePrefix = true
|
||||
|
||||
await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.title', false, true, [], 20, 0)
|
||||
|
||||
const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0]
|
||||
const displayTitleExpression = findOptions.attributes.include[0][0].val
|
||||
|
||||
expect(displayTitleExpression).to.include("bs.id IN ('series-id-''quoted''')")
|
||||
expect(displayTitleExpression).to.include('libraryItem.titleIgnorePrefix')
|
||||
})
|
||||
|
||||
it('should use postgres-safe join alias for sequence sorting', () => {
|
||||
Database.sequelize = createSequelizeStub('postgres', modelStubs)
|
||||
|
||||
const order = libraryItemsBookFilters.getOrder('sequence', false, false)
|
||||
const expression = order[0][0].val
|
||||
|
||||
expect(expression).to.include('"series->bookSeries"."sequence"')
|
||||
expect(expression).to.include('CASE WHEN BTRIM')
|
||||
})
|
||||
|
||||
it('should log generated query when findAndCountAll fails', async () => {
|
||||
Database.sequelize = createSequelizeStub('postgres', modelStubs)
|
||||
modelStubs.book.findAndCountAll.rejects(new Error('boom'))
|
||||
const errorStub = sinon.stub(Logger, 'error')
|
||||
|
||||
try {
|
||||
await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, false, [], 20, 0)
|
||||
expect.fail('Expected getFilteredLibraryItems to throw')
|
||||
} catch (error) {
|
||||
expect(error.message).to.equal('boom')
|
||||
}
|
||||
|
||||
expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll failed: boom')).to.equal(true)
|
||||
expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll query:')).to.equal(true)
|
||||
})
|
||||
|
||||
it('should use postgres-safe books->bookSeries alias in collapse-series selector query', async () => {
|
||||
Database.sequelize = createSequelizeStub('postgres', modelStubs)
|
||||
modelStubs.series.findAll.resolves([])
|
||||
|
||||
await libraryItemsBookFilters.getCollapseSeriesBooksToExclude({ where: {}, include: [] }, null)
|
||||
|
||||
const findAllOptions = modelStubs.series.findAll.firstCall.args[0]
|
||||
const orderExpression = findAllOptions.order[0].val
|
||||
|
||||
expect(orderExpression).to.include('"books->bookSeries"."sequence"')
|
||||
})
|
||||
})
|
||||
75
test/server/utils/queries/libraryItemsPodcastFilters.test.js
Normal file
75
test/server/utils/queries/libraryItemsPodcastFilters.test.js
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
const { expect } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
|
||||
const Database = require('../../../../server/Database')
|
||||
const libraryItemsPodcastFilters = require('../../../../server/utils/queries/libraryItemsPodcastFilters')
|
||||
|
||||
describe('libraryItemsPodcastFilters dialect behavior', () => {
|
||||
let originalSequelize
|
||||
let originalServerSettings
|
||||
|
||||
beforeEach(() => {
|
||||
originalSequelize = Database.sequelize
|
||||
originalServerSettings = global.ServerSettings
|
||||
global.ServerSettings = { sortingIgnorePrefix: false }
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
Database.sequelize = originalSequelize
|
||||
global.ServerSettings = originalServerSettings
|
||||
sinon.restore()
|
||||
})
|
||||
|
||||
it('should build postgres json tag permission predicate', () => {
|
||||
Database.sequelize = { getDialect: () => 'postgres' }
|
||||
|
||||
const { podcastWhere, replacements } = libraryItemsPodcastFilters.getUserPermissionPodcastWhereQuery({
|
||||
canAccessExplicitContent: true,
|
||||
permissions: {
|
||||
accessAllTags: false,
|
||||
itemTagsSelected: ['fiction'],
|
||||
selectedTagsNotAccessible: false
|
||||
}
|
||||
})
|
||||
|
||||
expect(replacements.userTagsSelected).to.deep.equal(['fiction'])
|
||||
expect(podcastWhere[0].attribute.val).to.include('jsonb_array_elements_text')
|
||||
})
|
||||
|
||||
it('should build sqlite no-case author sort expression', () => {
|
||||
Database.sequelize = { getDialect: () => 'sqlite' }
|
||||
|
||||
const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false)
|
||||
|
||||
expect(order[0][0].val).to.include('podcast.author COLLATE NOCASE')
|
||||
})
|
||||
|
||||
it('should build postgres lower author sort expression', () => {
|
||||
Database.sequelize = { getDialect: () => 'postgres' }
|
||||
|
||||
const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false)
|
||||
|
||||
expect(order[0][0].val).to.include('LOWER(podcast.author)')
|
||||
})
|
||||
|
||||
it('should use postgres json duration extraction in podcast stats query', async () => {
|
||||
const queryStub = sinon.stub()
|
||||
queryStub.onFirstCall().resolves([[{ totalSize: 1000 }]])
|
||||
queryStub.onSecondCall().resolves([[{ totalDuration: '12.3', totalItems: '4', numAudioFiles: '9' }]])
|
||||
|
||||
Database.sequelize = {
|
||||
getDialect: () => 'postgres',
|
||||
query: queryStub
|
||||
}
|
||||
|
||||
const result = await libraryItemsPodcastFilters.getPodcastLibraryStats('library-1')
|
||||
|
||||
expect(queryStub.secondCall.args[0]).to.include('NULLIF(pe.audioFile::jsonb #>>')
|
||||
expect(result).to.deep.equal({
|
||||
totalSize: 1000,
|
||||
totalDuration: '12.3',
|
||||
numAudioFiles: '9',
|
||||
totalItems: '4'
|
||||
})
|
||||
})
|
||||
})
|
||||
107
test/server/utils/queries/seriesFilters.test.js
Normal file
107
test/server/utils/queries/seriesFilters.test.js
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
const { expect } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
|
||||
const Database = require('../../../../server/Database')
|
||||
const seriesFilters = require('../../../../server/utils/queries/seriesFilters')
|
||||
const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters')
|
||||
|
||||
function encodedFilter(group, value) {
|
||||
return `${group}.${encodeURIComponent(Buffer.from(value).toString('base64'))}`
|
||||
}
|
||||
|
||||
describe('seriesFilters dialect behavior', () => {
|
||||
let originalSequelize
|
||||
let originalServerSettings
|
||||
let modelStubs
|
||||
|
||||
function setDialect(dialect) {
|
||||
Database.sequelize = {
|
||||
getDialect: () => dialect,
|
||||
random: sinon.stub(),
|
||||
models: modelStubs
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
originalSequelize = Database.sequelize
|
||||
originalServerSettings = global.ServerSettings
|
||||
modelStubs = {
|
||||
series: {
|
||||
findAndCountAll: sinon.stub().resolves({ rows: [], count: 0 })
|
||||
},
|
||||
bookSeries: {},
|
||||
book: {},
|
||||
libraryItem: {},
|
||||
author: {},
|
||||
feed: {}
|
||||
}
|
||||
|
||||
global.ServerSettings = { sortingIgnorePrefix: false }
|
||||
sinon.stub(libraryItemsBookFilters, 'getUserPermissionBookWhereQuery').returns({
|
||||
bookWhere: [],
|
||||
replacements: {}
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
Database.sequelize = originalSequelize
|
||||
global.ServerSettings = originalServerSettings
|
||||
sinon.restore()
|
||||
})
|
||||
|
||||
it('should build postgres progress filter with TRUE and FALSE literals', async () => {
|
||||
setDialect('postgres')
|
||||
|
||||
await seriesFilters.getFilteredSeries(
|
||||
{ id: 'library-1', settings: { hideSingleBookSeries: false } },
|
||||
{ id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } },
|
||||
encodedFilter('progress', 'not-started'),
|
||||
'name',
|
||||
false,
|
||||
[],
|
||||
10,
|
||||
0
|
||||
)
|
||||
|
||||
const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0]
|
||||
const progressWhere = findOptions.where[1]
|
||||
|
||||
expect(progressWhere.attribute.val).to.include('mp.isFinished = TRUE')
|
||||
})
|
||||
|
||||
it('should build sqlite no-case name sort expression', async () => {
|
||||
setDialect('sqlite')
|
||||
|
||||
await seriesFilters.getFilteredSeries(
|
||||
{ id: 'library-1', settings: { hideSingleBookSeries: false } },
|
||||
{ id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } },
|
||||
null,
|
||||
'name',
|
||||
false,
|
||||
[],
|
||||
10,
|
||||
0
|
||||
)
|
||||
|
||||
const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0]
|
||||
expect(findOptions.order[0][0].val).to.equal('series.name COLLATE NOCASE')
|
||||
})
|
||||
|
||||
it('should build postgres lower-case name sort expression', async () => {
|
||||
setDialect('postgres')
|
||||
|
||||
await seriesFilters.getFilteredSeries(
|
||||
{ id: 'library-1', settings: { hideSingleBookSeries: false } },
|
||||
{ id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } },
|
||||
null,
|
||||
'name',
|
||||
false,
|
||||
[],
|
||||
10,
|
||||
0
|
||||
)
|
||||
|
||||
const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0]
|
||||
expect(findOptions.order[0][0].val).to.equal('LOWER(series.name)')
|
||||
})
|
||||
})
|
||||
68
test/server/utils/sqlDialectHelpers.test.js
Normal file
68
test/server/utils/sqlDialectHelpers.test.js
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
const { expect } = require('chai')
|
||||
|
||||
const {
|
||||
booleanLiteral,
|
||||
noCaseSortExpression,
|
||||
coalesceFunctionName,
|
||||
jsonArrayContainsAny,
|
||||
jsonArrayContainsValue,
|
||||
jsonArrayExpand,
|
||||
jsonPathText,
|
||||
jsonPathNumber,
|
||||
safeTextToDoubleExpression,
|
||||
safeTextToIntegerExpression
|
||||
} = require('../../../server/utils/sqlDialectHelpers')
|
||||
|
||||
const sqlite = {
|
||||
getDialect: () => 'sqlite'
|
||||
}
|
||||
|
||||
const postgres = {
|
||||
getDialect: () => 'postgres'
|
||||
}
|
||||
|
||||
describe('sqlDialectHelpers', () => {
|
||||
it('should return sqlite and postgres boolean literals', () => {
|
||||
expect(booleanLiteral(true, sqlite)).to.equal('1')
|
||||
expect(booleanLiteral(false, sqlite)).to.equal('0')
|
||||
expect(booleanLiteral(true, postgres)).to.equal('TRUE')
|
||||
expect(booleanLiteral(false, postgres)).to.equal('FALSE')
|
||||
})
|
||||
|
||||
it('should generate case-insensitive sort expressions', () => {
|
||||
expect(noCaseSortExpression('name', sqlite)).to.equal('name COLLATE NOCASE')
|
||||
expect(noCaseSortExpression('name', postgres)).to.equal('LOWER(name)')
|
||||
})
|
||||
|
||||
it('should choose the correct null-coalescing function name', () => {
|
||||
expect(coalesceFunctionName(sqlite)).to.equal('IFNULL')
|
||||
expect(coalesceFunctionName(postgres)).to.equal('COALESCE')
|
||||
})
|
||||
|
||||
it('should generate array membership count queries per dialect', () => {
|
||||
expect(jsonArrayContainsAny('tags', 'selectedTags', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:selectedTags))')
|
||||
expect(jsonArrayContainsAny('tags', 'selectedTags', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:selectedTags))")
|
||||
|
||||
expect(jsonArrayContainsValue('tags', 'tag', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value = :tag)')
|
||||
expect(jsonArrayContainsValue('tags', 'tag', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :tag)")
|
||||
})
|
||||
|
||||
it('should generate json array expansion and path extraction by dialect', () => {
|
||||
expect(jsonArrayExpand('books.tags', sqlite)).to.equal('json_each(books.tags)')
|
||||
expect(jsonArrayExpand('books.tags', postgres)).to.equal("jsonb_array_elements_text(COALESCE(books.tags::jsonb, '[]'::jsonb)) AS json_each(value)")
|
||||
|
||||
expect(jsonPathText('payload', ['metadata', 'filename'], sqlite)).to.equal("json_extract(payload, '$.metadata.filename')")
|
||||
expect(jsonPathText('payload', ['metadata', 'filename'], postgres)).to.equal("payload::jsonb #>> '{metadata,filename}'")
|
||||
|
||||
expect(jsonPathNumber('payload', ['duration'], sqlite)).to.equal("json_extract(payload, '$.duration')")
|
||||
expect(jsonPathNumber('payload', ['duration'], postgres)).to.equal("NULLIF(payload::jsonb #>> '{duration}', '')::double precision")
|
||||
})
|
||||
|
||||
it('should generate safe numeric cast expressions for sequence-like text', () => {
|
||||
expect(safeTextToDoubleExpression('sequence', sqlite)).to.equal('CAST(sequence AS FLOAT)')
|
||||
expect(safeTextToDoubleExpression('"bookSeries"."sequence"', postgres)).to.equal("CASE WHEN BTRIM(\"bookSeries\".\"sequence\") ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(\"bookSeries\".\"sequence\")::double precision ELSE NULL END")
|
||||
|
||||
expect(safeTextToIntegerExpression('publishedYear', sqlite)).to.equal('CAST(publishedYear AS INTEGER)')
|
||||
expect(safeTextToIntegerExpression('book.publishedYear', postgres)).to.equal("CASE WHEN BTRIM(book.publishedYear) ~ '^[+-]?\\d+$' THEN BTRIM(book.publishedYear)::integer ELSE NULL END")
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue