This commit is contained in:
Kevin GATERA 2026-08-02 22:41:27 +00:00 committed by GitHub
commit cdfff4dc8b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
57 changed files with 3837 additions and 508 deletions

View file

@ -0,0 +1,45 @@
name: Homelab Docker Build
on:
push:
branches:
- homelab
- homelab-postgres
workflow_dispatch:
permissions:
contents: read
packages: write
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: Check out
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push homelab image
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64
push: true
tags: |
ghcr.io/kevingatera/audiobookshelf:${{ github.ref_name }}
ghcr.io/kevingatera/audiobookshelf:${{ github.ref_name }}-${{ github.sha }}
labels: |
org.opencontainers.image.source=https://github.com/kevingatera/audiobookshelf
org.opencontainers.image.revision=${{ github.sha }}

View file

@ -50,6 +50,7 @@ ARG NUSQLITE3_PATH
RUN apk add --no-cache --update \ RUN apk add --no-cache --update \
tzdata \ tzdata \
ffmpeg \ ffmpeg \
postgresql-client \
tini tini
WORKDIR /app WORKDIR /app

167
package-lock.json generated
View file

@ -23,6 +23,8 @@
"p-throttle": "^4.1.1", "p-throttle": "^4.1.1",
"passport": "^0.6.0", "passport": "^0.6.0",
"passport-jwt": "^4.0.1", "passport-jwt": "^4.0.1",
"pg": "^8.19.0",
"pg-hstore": "^2.3.4",
"semver": "^7.6.3", "semver": "^7.6.3",
"sequelize": "^6.35.2", "sequelize": "^6.35.2",
"socket.io": "^4.5.4", "socket.io": "^4.5.4",
@ -4138,10 +4140,106 @@
"resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz",
"integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg=="
}, },
"node_modules/pg": {
"version": "8.22.0",
"resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz",
"integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==",
"license": "MIT",
"dependencies": {
"pg-connection-string": "^2.14.0",
"pg-pool": "^3.14.0",
"pg-protocol": "^1.15.0",
"pg-types": "2.2.0",
"pgpass": "1.0.5"
},
"engines": {
"node": ">= 16.0.0"
},
"optionalDependencies": {
"pg-cloudflare": "^1.4.0"
},
"peerDependencies": {
"pg-native": ">=3.0.1"
},
"peerDependenciesMeta": {
"pg-native": {
"optional": true
}
}
},
"node_modules/pg-cloudflare": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz",
"integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==",
"license": "MIT",
"optional": true
},
"node_modules/pg-connection-string": { "node_modules/pg-connection-string": {
"version": "2.6.1", "version": "2.14.0",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz",
"integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==",
"license": "MIT"
},
"node_modules/pg-hstore": {
"version": "2.3.4",
"resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz",
"integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==",
"license": "MIT",
"dependencies": {
"underscore": "^1.13.1"
},
"engines": {
"node": ">= 0.8.x"
}
},
"node_modules/pg-int8": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
"license": "ISC",
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/pg-pool": {
"version": "3.14.0",
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
"integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
"license": "MIT",
"peerDependencies": {
"pg": ">=8.0"
}
},
"node_modules/pg-protocol": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz",
"integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==",
"license": "MIT"
},
"node_modules/pg-types": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
"license": "MIT",
"dependencies": {
"pg-int8": "1.0.1",
"postgres-array": "~2.0.0",
"postgres-bytea": "~1.0.0",
"postgres-date": "~1.0.4",
"postgres-interval": "^1.1.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/pgpass": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
"license": "MIT",
"dependencies": {
"split2": "^4.1.0"
}
}, },
"node_modules/picocolors": { "node_modules/picocolors": {
"version": "1.0.0", "version": "1.0.0",
@ -4173,6 +4271,45 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/postgres-array": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/postgres-bytea": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
"integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/postgres-date": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/postgres-interval": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
"license": "MIT",
"dependencies": {
"xtend": "^4.0.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/prebuild-install": { "node_modules/prebuild-install": {
"version": "7.1.3", "version": "7.1.3",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
@ -4946,6 +5083,15 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/split2": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
"license": "ISC",
"engines": {
"node": ">= 10.x"
}
},
"node_modules/sprintf-js": { "node_modules/sprintf-js": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@ -5265,6 +5411,12 @@
"integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
"dev": true "dev": true
}, },
"node_modules/underscore": {
"version": "1.13.8",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz",
"integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==",
"license": "MIT"
},
"node_modules/unique-filename": { "node_modules/unique-filename": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz",
@ -5476,6 +5628,15 @@
"node": ">=4.0" "node": ">=4.0"
} }
}, },
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
"license": "MIT",
"engines": {
"node": ">=0.4"
}
},
"node_modules/y18n": { "node_modules/y18n": {
"version": "5.0.8", "version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

View file

@ -16,6 +16,7 @@
"docker": "docker buildx build --platform linux/amd64,linux/arm64 --push . -t advplyr/audiobookshelf", "docker": "docker buildx build --platform linux/amd64,linux/arm64 --push . -t advplyr/audiobookshelf",
"docker-amd64-local": "docker buildx build --platform linux/amd64 --load . -t advplyr/audiobookshelf-amd64-local", "docker-amd64-local": "docker buildx build --platform linux/amd64 --load . -t advplyr/audiobookshelf-amd64-local",
"docker-arm64-local": "docker buildx build --platform linux/arm64 --load . -t advplyr/audiobookshelf-arm64-local", "docker-arm64-local": "docker buildx build --platform linux/arm64 --load . -t advplyr/audiobookshelf-arm64-local",
"migrate-sqlite-to-postgres": "node server/scripts/migrateSqliteToPostgres.js",
"deploy-linux": "node deploy/linux", "deploy-linux": "node deploy/linux",
"test": "mocha", "test": "mocha",
"coverage": "nyc mocha" "coverage": "nyc mocha"
@ -52,6 +53,8 @@
"p-throttle": "^4.1.1", "p-throttle": "^4.1.1",
"passport": "^0.6.0", "passport": "^0.6.0",
"passport-jwt": "^4.0.1", "passport-jwt": "^4.0.1",
"pg": "^8.19.0",
"pg-hstore": "^2.3.4",
"semver": "^7.6.3", "semver": "^7.6.3",
"sequelize": "^6.35.2", "sequelize": "^6.35.2",
"socket.io": "^4.5.4", "socket.io": "^4.5.4",

View file

@ -14,6 +14,7 @@ class Database {
constructor() { constructor() {
this.sequelize = null this.sequelize = null
this.dbPath = null this.dbPath = null
this.dialect = 'sqlite'
this.isNew = false // New absdatabase.sqlite created this.isNew = false // New absdatabase.sqlite created
this.hasRootUser = false // Used to show initialization page in web ui this.hasRootUser = false // Used to show initialization page in web ui
@ -162,11 +163,37 @@ class Database {
return this.models.device return this.models.device
} }
/**
* @returns {'sqlite'|'postgres'}
*/
getConfiguredDialect() {
const explicitDialect = process.env.DB_DIALECT?.trim()?.toLowerCase()
if (explicitDialect === 'postgres' || explicitDialect === 'sqlite') {
return explicitDialect
}
const databaseUrl = process.env.DATABASE_URL || ''
if (databaseUrl.startsWith('postgres://') || databaseUrl.startsWith('postgresql://')) {
return 'postgres'
}
return 'sqlite'
}
isSqliteDialect() {
return this.dialect === 'sqlite'
}
isPostgresDialect() {
return this.dialect === 'postgres'
}
/** /**
* Check if db file exists * Check if db file exists
* @returns {boolean} * @returns {boolean}
*/ */
async checkHasDb() { async checkHasDb() {
if (!this.isSqliteDialect()) return true
if (!(await fs.pathExists(this.dbPath))) { if (!(await fs.pathExists(this.dbPath))) {
Logger.info(`[Database] absdatabase.sqlite not found at ${this.dbPath}`) Logger.info(`[Database] absdatabase.sqlite not found at ${this.dbPath}`)
return false return false
@ -174,20 +201,39 @@ class Database {
return true return true
} }
/**
* Check if any user tables exist (for networked dialects)
* @returns {Promise<boolean>}
*/
async checkHasTables() {
const queryInterface = this.sequelize.getQueryInterface()
const tables = await queryInterface.showAllTables()
return Array.isArray(tables) && tables.length > 0
}
/** /**
* Connect to db, build models and run migrations * Connect to db, build models and run migrations
* @param {boolean} [force=false] Used for testing, drops & re-creates all tables * @param {boolean} [force=false] Used for testing, drops & re-creates all tables
*/ */
async init(force = false) { async init(force = false) {
this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') this.dialect = this.getConfiguredDialect()
if (this.isSqliteDialect()) {
// First check if this is a new database this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite')
this.isNew = !(await this.checkHasDb()) || force // First check if this is a new database
this.isNew = !(await this.checkHasDb()) || force
} else {
this.dbPath = process.env.DATABASE_URL || null
this.isNew = !!force
}
if (!(await this.connect())) { if (!(await this.connect())) {
throw new Error('Database connection failed') throw new Error('Database connection failed')
} }
if (this.isPostgresDialect() && !force) {
this.isNew = !(await this.checkHasTables())
}
try { try {
const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath) const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath)
await migrationManager.init(packageJson.version) await migrationManager.init(packageJson.version)
@ -214,7 +260,11 @@ class Database {
* @returns {boolean} * @returns {boolean}
*/ */
async connect() { async connect() {
Logger.info(`[Database] Initializing db at "${this.dbPath}"`) if (this.isSqliteDialect()) {
Logger.info(`[Database] Initializing sqlite db at "${this.dbPath}"`)
} else {
Logger.info(`[Database] Initializing postgres db connection`)
}
let logging = false let logging = false
let benchmark = false let benchmark = false
@ -229,13 +279,27 @@ class Database {
benchmark = true benchmark = true
} }
this.sequelize = new Sequelize({ if (this.isSqliteDialect()) {
dialect: 'sqlite', this.sequelize = new Sequelize({
storage: this.dbPath, dialect: 'sqlite',
logging: logging, storage: this.dbPath,
benchmark: benchmark, logging: logging,
transactionType: 'IMMEDIATE' benchmark: benchmark,
}) transactionType: 'IMMEDIATE'
})
} else {
if (!process.env.DATABASE_URL) {
Logger.error(`[Database] DATABASE_URL is required when DB_DIALECT=postgres`)
return false
}
this.sequelize = new Sequelize(process.env.DATABASE_URL, {
dialect: 'postgres',
logging: logging,
benchmark: benchmark,
quoteIdentifiers: false
})
}
// Helper function // Helper function
this.sequelize.uppercaseFirst = (str) => (str ? `${str[0].toUpperCase()}${str.substr(1)}` : '') this.sequelize.uppercaseFirst = (str) => (str ? `${str[0].toUpperCase()}${str.substr(1)}` : '')
@ -243,34 +307,36 @@ class Database {
try { try {
await this.sequelize.authenticate() await this.sequelize.authenticate()
// Set SQLite pragmas from environment variables if (this.isSqliteDialect()) {
const allowedPragmas = [ // Set SQLite pragmas from environment variables
{ name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' }, const allowedPragmas = [
{ name: 'cache_size', env: 'SQLITE_CACHE_SIZE' }, { name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' },
{ name: 'temp_store', env: 'SQLITE_TEMP_STORE' } { name: 'cache_size', env: 'SQLITE_CACHE_SIZE' },
] { name: 'temp_store', env: 'SQLITE_TEMP_STORE' }
]
for (const pragma of allowedPragmas) { for (const pragma of allowedPragmas) {
const value = process.env[pragma.env] const value = process.env[pragma.env]
if (value !== undefined) { if (value !== undefined) {
try { try {
Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`) Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`)
await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`) await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`)
const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`) const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`)
Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result) Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result)
} catch (error) { } catch (error) {
Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error) Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error)
}
} }
} }
}
if (process.env.NUSQLITE3_PATH) { if (process.env.NUSQLITE3_PATH) {
await this.loadExtension(process.env.NUSQLITE3_PATH) await this.loadExtension(process.env.NUSQLITE3_PATH)
Logger.info(`[Database] Db supports unaccent and unicode foldings`) Logger.info(`[Database] Db supports unaccent and unicode foldings`)
this.supportsUnaccent = true this.supportsUnaccent = true
this.supportsUnicodeFoldings = true this.supportsUnicodeFoldings = true
}
} }
Logger.info(`[Database] Db connection was successful`) Logger.info(`[Database] Db connection was successful (${this.dialect})`)
return true return true
} catch (error) { } catch (error) {
Logger.error(`[Database] Failed to connect to db`, error) Logger.error(`[Database] Failed to connect to db`, error)
@ -307,7 +373,7 @@ class Database {
* Disconnect from db * Disconnect from db
*/ */
async disconnect() { async disconnect() {
Logger.info(`[Database] Disconnecting sqlite db`) Logger.info(`[Database] Disconnecting ${this.dialect} db`)
await this.sequelize.close() await this.sequelize.close()
} }
@ -315,7 +381,7 @@ class Database {
* Reconnect to db and init * Reconnect to db and init
*/ */
async reconnect() { async reconnect() {
Logger.info(`[Database] Reconnecting sqlite db`) Logger.info(`[Database] Reconnecting ${this.dialect} db`)
await this.init() await this.init()
} }
@ -346,6 +412,11 @@ class Database {
require('./models/CustomMetadataProvider').init(this.sequelize) require('./models/CustomMetadataProvider').init(this.sequelize)
require('./models/MediaItemShare').init(this.sequelize) require('./models/MediaItemShare').init(this.sequelize)
if (this.isPostgresDialect() && !force && !this.isNew) {
Logger.info('[Database] Skipping sequelize.sync for existing postgres schema')
return Promise.resolve()
}
return this.sequelize.sync({ force, alter: false }) return this.sequelize.sync({ force, alter: false })
} }
@ -368,7 +439,7 @@ class Database {
* Loads most of the data from the database. This is a temporary solution. * Loads most of the data from the database. This is a temporary solution.
*/ */
async loadData() { async loadData() {
if (this.isNew && (await dbMigration.checkShouldMigrate())) { if (this.isSqliteDialect() && this.isNew && (await dbMigration.checkShouldMigrate())) {
Logger.info(`[Database] New database was created and old database was detected - migrating old to new`) Logger.info(`[Database] New database was created and old database was detected - migrating old to new`)
await dbMigration.migrate(this.models) await dbMigration.migrate(this.models)
} }
@ -441,9 +512,19 @@ class Database {
return this.models.setting.updateSettingObj(settings.toJSON()) return this.models.setting.updateSettingObj(settings.toJSON())
} }
getPlaybackSessions(where = null) { getPlaybackSessions(where = null, options = null) {
if (!this.sequelize) return false if (!this.sequelize) return false
return this.models.playbackSession.getOldPlaybackSessions(where) return this.models.playbackSession.getOldPlaybackSessions(where, options || undefined)
}
countPlaybackSessions(where = null) {
if (!this.sequelize) return false
return this.models.playbackSession.countWithWhere(where)
}
getPlaybackSessionsForStats(where = null) {
if (!this.sequelize) return false
return this.models.playbackSession.getPlaybackSessionsForStats(where)
} }
getPlaybackSession(sessionId) { getPlaybackSession(sessionId) {
@ -847,6 +928,14 @@ WHERE EXISTS (
* It adds triggers to update libraryItems.title[IgnorePrefix] when (books|podcasts).title[IgnorePrefix] is updated * It adds triggers to update libraryItems.title[IgnorePrefix] when (books|podcasts).title[IgnorePrefix] is updated
*/ */
async addTriggers() { async addTriggers() {
if (this.isPostgresDialect()) {
return this.addPostgresTriggers()
}
if (!this.isSqliteDialect()) {
Logger.info(`[Database] Skipping sqlite-only triggers for dialect ${this.dialect}`)
return
}
await this.addTriggerIfNotExists('books', 'title', 'id', 'libraryItems', 'title', 'mediaId') await this.addTriggerIfNotExists('books', 'title', 'id', 'libraryItems', 'title', 'mediaId')
await this.addTriggerIfNotExists('books', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId') await this.addTriggerIfNotExists('books', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId')
await this.addTriggerIfNotExists('podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId') await this.addTriggerIfNotExists('podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId')
@ -944,6 +1033,121 @@ WHERE EXISTS (
await addAuthorsUpdateTriggerIfNotExists() await addAuthorsUpdateTriggerIfNotExists()
} }
/**
* Postgres equivalent of the sqlite libraryItems denormalization triggers above.
* Identifiers are unquoted (and therefore folded to lowercase by postgres) to match
* the quoteIdentifiers: false strategy used for the postgres dialect.
*/
async addPostgresTriggers() {
Logger.info('[Database] Adding postgres denormalization triggers')
await this.addPostgresTitleTriggerIfNotExists('books', 'title')
await this.addPostgresTitleTriggerIfNotExists('books', 'titleIgnorePrefix')
await this.addPostgresTitleTriggerIfNotExists('podcasts', 'title')
await this.addPostgresTitleTriggerIfNotExists('podcasts', 'titleIgnorePrefix')
await this.addPostgresAuthorNamesTriggersIfNotExist()
}
async postgresTriggerExists(triggerName) {
const [[{ count }]] = await this.sequelize.query(`SELECT COUNT(*) as count FROM pg_trigger WHERE NOT tgisinternal AND tgname = '${triggerName}'`)
return Number(count) > 0
}
async addPostgresTitleTriggerIfNotExists(sourceTable, sourceColumn) {
const foldedColumn = sourceColumn.toLowerCase()
const action = `update_libraryItems_${sourceColumn}`
const fromSource = sourceTable === 'books' ? '' : `_from_${sourceTable}_${sourceColumn}`
const triggerName = this.convertToSnakeCase(`${action}${fromSource}`)
const functionName = `${triggerName}_fn`
if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists
Logger.info(`[Database] Adding trigger ${triggerName}`)
await this.sequelize.query(`
CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$
BEGIN
UPDATE libraryitems
SET ${foldedColumn} = NEW.${foldedColumn}
WHERE mediaid = NEW.id;
RETURN NEW;
END;
$func$ LANGUAGE plpgsql
`)
await this.sequelize.query(`
CREATE TRIGGER ${triggerName}
AFTER UPDATE OF ${foldedColumn} ON ${sourceTable}
FOR EACH ROW
EXECUTE FUNCTION ${functionName}()
`)
}
async addPostgresAuthorNamesTriggersIfNotExist() {
// string_agg is the postgres equivalent of sqlite GROUP_CONCAT; both return NULL for an empty set
const authorNamesSubQuery = (bookIdExpression) => `
SELECT string_agg(authors.name, ', ' ORDER BY bookauthors.createdat ASC), string_agg(authors.lastfirst, ', ' ORDER BY bookauthors.createdat ASC)
FROM authors JOIN bookauthors ON authors.id = bookauthors.authorid
WHERE bookauthors.bookid = ${bookIdExpression}
`
const addBookAuthorsTriggerIfNotExists = async (action) => {
const modifiedRecord = action === 'delete' ? 'OLD' : 'NEW'
const triggerName = this.convertToSnakeCase(`update_libraryItems_authorNames_on_bookAuthors_${action}`)
const functionName = `${triggerName}_fn`
if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists
Logger.info(`[Database] Adding trigger ${triggerName}`)
await this.sequelize.query(`
CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$
BEGIN
UPDATE libraryitems
SET (authornamesfirstlast, authornameslastfirst) = (${authorNamesSubQuery(`${modifiedRecord}.bookid`)})
WHERE mediaid = ${modifiedRecord}.bookid;
RETURN ${modifiedRecord};
END;
$func$ LANGUAGE plpgsql
`)
await this.sequelize.query(`
CREATE TRIGGER ${triggerName}
AFTER ${action.toUpperCase()} ON bookauthors
FOR EACH ROW
EXECUTE FUNCTION ${functionName}()
`)
}
const addAuthorsUpdateTriggerIfNotExists = async () => {
const triggerName = this.convertToSnakeCase('update_libraryItems_authorNames_on_authors_update')
const functionName = `${triggerName}_fn`
if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists
Logger.info(`[Database] Adding trigger ${triggerName}`)
await this.sequelize.query(`
CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$
BEGIN
UPDATE libraryitems
SET (authornamesfirstlast, authornameslastfirst) = (${authorNamesSubQuery('libraryitems.mediaid')})
WHERE mediaid IN (SELECT bookid FROM bookauthors WHERE authorid = NEW.id);
RETURN NEW;
END;
$func$ LANGUAGE plpgsql
`)
await this.sequelize.query(`
CREATE TRIGGER ${triggerName}
AFTER UPDATE OF name ON authors
FOR EACH ROW
EXECUTE FUNCTION ${functionName}()
`)
}
await addBookAuthorsTriggerIfNotExists('insert')
await addBookAuthorsTriggerIfNotExists('delete')
await addAuthorsUpdateTriggerIfNotExists()
}
convertToSnakeCase(str) { convertToSnakeCase(str) {
return str.replace(/([A-Z])/g, '_$1').toLowerCase() return str.replace(/([A-Z])/g, '_$1').toLowerCase()
} }
@ -954,6 +1158,7 @@ WHERE EXISTS (
this.supportsUnaccent = supportsUnaccent this.supportsUnaccent = supportsUnaccent
this.query = query this.query = query
this.hasAccents = false this.hasAccents = false
this.dialect = sequelize.getDialect()
} }
/** /**
@ -989,9 +1194,10 @@ WHERE EXISTS (
*/ */
matchExpression(column) { matchExpression(column) {
const pattern = this.sequelize.escape(`%${this.query}%`) const pattern = this.sequelize.escape(`%${this.query}%`)
if (!this.supportsUnaccent) return `${column} LIKE ${pattern}` const likeOperator = this.dialect === 'postgres' ? 'ILIKE' : 'LIKE'
if (!this.supportsUnaccent) return `${column} ${likeOperator} ${pattern}`
const normalizedColumn = this.hasAccents ? column : this.normalize(column) const normalizedColumn = this.hasAccents ? column : this.normalize(column)
return `${normalizedColumn} LIKE ${pattern}` return `${normalizedColumn} ${likeOperator} ${pattern}`
} }
} }
} }

View file

@ -479,14 +479,27 @@ class Server {
// Remove series from hide from continue listening that no longer exist // Remove series from hide from continue listening that no longer exist
try { try {
const users = await Database.sequelize.query(`SELECT u.id, u.username, u.extraData, json_group_array(value) AS seriesIdsToRemove FROM users u, json_each(u.extraData->"seriesHideFromContinueListening") LEFT JOIN series se ON se.id = value WHERE se.id IS NULL GROUP BY u.id;`, { const users = await Database.userModel.findAll({
model: Database.userModel, attributes: ['id', 'username', 'extraData']
type: Sequelize.QueryTypes.SELECT
}) })
for (const user of users) { for (const user of users) {
const extraData = JSON.parse(user.extraData) const extraData = typeof user.extraData === 'string' ? JSON.parse(user.extraData || '{}') : user.extraData || {}
const existingSeriesIds = extraData.seriesHideFromContinueListening const existingSeriesIds = Array.isArray(extraData.seriesHideFromContinueListening) ? extraData.seriesHideFromContinueListening : []
const seriesIdsToRemove = JSON.parse(user.dataValues.seriesIdsToRemove) if (!existingSeriesIds.length) continue
const existingSeries = await Database.seriesModel.findAll({
attributes: ['id'],
where: {
id: {
[Sequelize.Op.in]: existingSeriesIds
}
}
})
const existingSeriesSet = new Set(existingSeries.map((series) => series.id))
const seriesIdsToRemove = existingSeriesIds.filter((seriesId) => !existingSeriesSet.has(seriesId))
if (!seriesIdsToRemove.length) continue
Logger.info(`[Server] Found ${seriesIdsToRemove.length} non-existent series in seriesHideFromContinueListening for user "${user.username}" - Removing (${seriesIdsToRemove.join(',')})`) Logger.info(`[Server] Found ${seriesIdsToRemove.length} non-existent series in seriesHideFromContinueListening for user "${user.username}" - Removing (${seriesIdsToRemove.join(',')})`)
const newExtraData = { const newExtraData = {
...extraData, ...extraData,

View file

@ -24,6 +24,7 @@ const libraryFilters = require('../utils/queries/libraryFilters')
const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters') const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters')
const authorFilters = require('../utils/queries/authorFilters') const authorFilters = require('../utils/queries/authorFilters')
const zipHelpers = require('../utils/zipHelpers') const zipHelpers = require('../utils/zipHelpers')
const { isPostgres, noCaseSortExpression, jsonArrayExpand } = require('../utils/sqlDialectHelpers')
/** /**
* @typedef RequestUserObject * @typedef RequestUserObject
@ -1039,9 +1040,9 @@ class LibraryController {
let order = undefined let order = undefined
const direction = payload.sortDesc ? 'DESC' : 'ASC' const direction = payload.sortDesc ? 'DESC' : 'ASC'
if (payload.sortBy === 'name') { if (payload.sortBy === 'name') {
order = [[Sequelize.literal('name COLLATE NOCASE'), direction]] order = [[Sequelize.literal(noCaseSortExpression('name', Database.sequelize)), direction]]
} else if (payload.sortBy === 'lastFirst') { } else if (payload.sortBy === 'lastFirst') {
order = [[Sequelize.literal('lastFirst COLLATE NOCASE'), direction]] order = [[Sequelize.literal(noCaseSortExpression('lastFirst', Database.sequelize)), direction]]
} else if (payload.sortBy === 'addedAt') { } else if (payload.sortBy === 'addedAt') {
order = [['createdAt', direction]] order = [['createdAt', direction]]
} else if (payload.sortBy === 'updatedAt') { } else if (payload.sortBy === 'updatedAt') {
@ -1338,16 +1339,23 @@ class LibraryController {
const fileExt = req.query.ext === 'abs' ? 'abs' : 'json' const fileExt = req.query.ext === 'abs' ? 'abs' : 'json'
const metadataFilename = `metadata.${fileExt}` const metadataFilename = `metadata.${fileExt}`
const metadataFilenameCountQuery = isPostgres(Database.sequelize)
? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{metadata,filename}' = :metadataFilename)`
: `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = :metadataFilename)`
const libraryItemsWithMetadata = await Database.libraryItemModel.findAll({ const libraryItemsWithMetadata = await Database.libraryItemModel.findAll({
attributes: ['id', 'libraryFiles'], attributes: ['id', 'libraryFiles'],
where: [ where: [
{ {
libraryId: req.library.id libraryId: req.library.id
}, },
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = "${metadataFilename}")`), { Sequelize.where(Sequelize.literal(metadataFilenameCountQuery), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}) })
] ],
replacements: {
metadataFilename
}
}) })
if (!libraryItemsWithMetadata.length) { if (!libraryItemsWithMetadata.length) {
Logger.info(`[LibraryController] No ${metadataFilename} files found to remove`) Logger.info(`[LibraryController] No ${metadataFilename} files found to remove`)

View file

@ -155,21 +155,23 @@ class MeController {
* @param {Response} res * @param {Response} res
*/ */
async getListeningSessions(req, res) { async getListeningSessions(req, res) {
const listeningSessions = await this.getUserListeningSessionsHelper(req.user.id) const startedAt = Date.now()
const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10 const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10
const page = toNumber(req.query.page, 0) const page = toNumber(req.query.page, 0)
const start = page * itemsPerPage Logger.debug(
const sessions = listeningSessions.slice(start, start + itemsPerPage) `[MeController] /api/me/listening-sessions user="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} start`
)
const payload = { const payload = await this.getUserListeningSessionsPageHelper(
total: listeningSessions.length, req.user.id,
numPages: Math.ceil(listeningSessions.length / itemsPerPage),
page, page,
itemsPerPage, itemsPerPage
sessions )
}
Logger.debug(
`[MeController] /api/me/listening-sessions user="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms`
)
res.json(payload) res.json(payload)
} }
@ -183,6 +185,7 @@ class MeController {
* @param {Response} res * @param {Response} res
*/ */
async getItemListeningSessions(req, res) { async getItemListeningSessions(req, res) {
const startedAt = Date.now()
const libraryItem = await Database.libraryItemModel.getExpandedById(req.params.libraryItemId) const libraryItem = await Database.libraryItemModel.getExpandedById(req.params.libraryItemId)
const episode = await Database.podcastEpisodeModel.findByPk(req.params.episodeId) const episode = await Database.podcastEpisodeModel.findByPk(req.params.episodeId)
@ -198,21 +201,23 @@ class MeController {
} }
const mediaItemId = episode?.id || libraryItem.mediaId const mediaItemId = episode?.id || libraryItem.mediaId
let listeningSessions = await this.getUserItemListeningSessionsHelper(req.user.id, mediaItemId)
const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10 const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10
const page = toNumber(req.query.page, 0) const page = toNumber(req.query.page, 0)
const start = page * itemsPerPage Logger.debug(
const sessions = listeningSessions.slice(start, start + itemsPerPage) `[MeController] /api/me/item/listening-sessions user="${req.user.id}" libraryItem="${req.params.libraryItemId}" episode="${req.params.episodeId || ''}" page=${page} itemsPerPage=${itemsPerPage} start`
)
const payload = { const payload = await this.getUserListeningSessionsPageHelper(
total: listeningSessions.length, req.user.id,
numPages: Math.ceil(listeningSessions.length / itemsPerPage),
page, page,
itemsPerPage, itemsPerPage,
sessions mediaItemId
} )
Logger.debug(
`[MeController] /api/me/item/listening-sessions user="${req.user.id}" libraryItem="${req.params.libraryItemId}" episode="${req.params.episodeId || ''}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms`
)
res.json(payload) res.json(payload)
} }
@ -226,7 +231,16 @@ class MeController {
* @param {Response} res * @param {Response} res
*/ */
async getListeningStats(req, res) { async getListeningStats(req, res) {
const listeningStats = await this.getUserListeningStatsHelpers(req.user.id) const startedAt = Date.now()
const minified = req.query.minified === '1' || req.query.minified === 'true'
Logger.debug(`[MeController] /api/me/listening-stats user="${req.user.id}" start`)
const listeningStats = await this.getUserListeningStatsHelpers(req.user.id, {
includeItems: !minified,
includeRecentSessions: !minified
})
Logger.debug(
`[MeController] /api/me/listening-stats user="${req.user.id}" minified=${minified} totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms`
)
res.json(listeningStats) res.json(listeningStats)
} }

View file

@ -432,14 +432,20 @@ class UserController {
* @param {Response} res * @param {Response} res
*/ */
async getListeningSessions(req, res) { async getListeningSessions(req, res) {
var listeningSessions = await this.getUserListeningSessionsHelper(req.params.id) const startedAt = Date.now()
const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10 const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10
const page = toNumber(req.query.page, 0) const page = toNumber(req.query.page, 0)
Logger.debug(
`[UserController] /api/users/${req.params.id}/listening-sessions reqUser="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} start`
)
const payload = await this.getUserListeningSessionsPageHelper(
req.params.id,
page,
itemsPerPage
)
const start = page * itemsPerPage
// Map user to sessions to match the format of the sessions endpoint // Map user to sessions to match the format of the sessions endpoint
const sessions = listeningSessions.slice(start, start + itemsPerPage).map((session) => { payload.sessions = payload.sessions.map((session) => {
return { return {
...session, ...session,
user: { user: {
@ -449,13 +455,9 @@ class UserController {
} }
}) })
const payload = { Logger.debug(
total: listeningSessions.length, `[UserController] /api/users/${req.params.id}/listening-sessions reqUser="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms`
numPages: Math.ceil(listeningSessions.length / itemsPerPage), )
page,
itemsPerPage,
sessions
}
res.json(payload) res.json(payload)
} }
@ -469,7 +471,18 @@ class UserController {
* @param {Response} res * @param {Response} res
*/ */
async getListeningStats(req, res) { async getListeningStats(req, res) {
var listeningStats = await this.getUserListeningStatsHelpers(req.params.id) const startedAt = Date.now()
const minified = req.query.minified === '1' || req.query.minified === 'true'
Logger.debug(
`[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" start`
)
var listeningStats = await this.getUserListeningStatsHelpers(req.params.id, {
includeItems: !minified,
includeRecentSessions: !minified
})
Logger.debug(
`[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" minified=${minified} totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms`
)
res.json(listeningStats) res.json(listeningStats)
} }

View file

@ -1,3 +1,4 @@
const childProcess = require('child_process')
const sqlite3 = require('sqlite3') const sqlite3 = require('sqlite3')
const Path = require('path') const Path = require('path')
const Logger = require('../Logger') const Logger = require('../Logger')
@ -47,6 +48,30 @@ class BackupManager {
return global.ServerSettings.maxBackupSize || Infinity return global.ServerSettings.maxBackupSize || Infinity
} }
get databaseBackupConfig() {
if (Database.isPostgresDialect()) {
return {
dialect: 'postgres',
entryName: 'absdatabase.postgres.dump'
}
}
return {
dialect: 'sqlite',
entryName: 'absdatabase.sqlite'
}
}
getBackupDialect(backup) {
if (backup.key === 'postgres') return 'postgres'
if (backup.key === 'sqlite' || !backup.key) return 'sqlite'
return null
}
getBackupEntryName(dialect) {
return dialect === 'postgres' ? 'absdatabase.postgres.dump' : 'absdatabase.sqlite'
}
async init() { async init() {
try { try {
const backupsDirExists = await fs.pathExists(this.backupPath) const backupsDirExists = await fs.pathExists(this.backupPath)
@ -130,13 +155,6 @@ class BackupManager {
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err)) await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(400).send('Failed to read backup file - backup might not be a valid .zip file') return res.status(400).send('Failed to read backup file - backup might not be a valid .zip file')
} }
if (!entries['absdatabase.sqlite']) {
Logger.error(`[BackupManager] Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.`)
await zip.close().catch(() => {})
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(500).send('Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.')
}
const detailsEntry = entries['details'] const detailsEntry = entries['details']
if (!detailsEntry) { if (!detailsEntry) {
Logger.error('[BackupManager] Invalid backup - missing details entry') Logger.error('[BackupManager] Invalid backup - missing details entry')
@ -151,10 +169,26 @@ class BackupManager {
return res.status(400).send('Invalid backup file - details entry too large') return res.status(400).send('Invalid backup file - details entry too large')
} }
const data = await zip.entryData('details') let backup
const details = data.toString('utf8').split('\n') try {
const data = await zip.entryData('details')
const details = data.toString('utf8').split('\n')
backup = new Backup({ details, fullPath: tempPath })
} catch (error) {
Logger.error(`[BackupManager] Invalid backup with no readable details file`, tempPath, error)
await zip.close().catch(() => {})
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(400).send('Invalid backup file. Missing readable details.')
}
const backup = new Backup({ details, fullPath: tempPath }) const backupDialect = this.getBackupDialect(backup)
const databaseEntryName = this.getBackupEntryName(backupDialect)
if (!backupDialect || !entries[databaseEntryName]) {
Logger.error(`[BackupManager] Invalid backup with no ${databaseEntryName} file - unsupported database backup.`)
await zip.close().catch(() => {})
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(500).send(`Invalid backup file. Does not include ${databaseEntryName}.`)
}
if (!backup.serverVersion) { if (!backup.serverVersion) {
Logger.error(`[BackupManager] Invalid backup with no server version - might be a backup created before version 2.0.0`) Logger.error(`[BackupManager] Invalid backup with no server version - might be a backup created before version 2.0.0`)
@ -204,8 +238,24 @@ class BackupManager {
const entries = await zip.entries() const entries = await zip.entries()
const backupDialect = this.getBackupDialect(backup)
const currentDialect = Database.isPostgresDialect() ? 'postgres' : 'sqlite'
if (!backupDialect) {
await zip.close()
return res.status(500).send('Invalid backup file. Unsupported database backup format.')
}
if (backupDialect !== currentDialect) {
await zip.close()
return res.status(400).send(`Cannot apply a ${backupDialect} backup while using the ${currentDialect} database.`)
}
if (backupDialect === 'postgres') {
return this.requestApplyPostgresBackup(apiCacheManager, backup, zip, entries, res)
}
// Ensure backup has an absdatabase.sqlite file // Ensure backup has an absdatabase.sqlite file
if (!Object.keys(entries).includes('absdatabase.sqlite')) { if (!Object.keys(entries).includes(this.getBackupEntryName(backupDialect))) {
Logger.error(`[BackupManager] Cannot apply old backup ${backup.fullPath}`) Logger.error(`[BackupManager] Cannot apply old backup ${backup.fullPath}`)
await zip.close() await zip.close()
return res.status(500).send('Invalid backup file. Does not include absdatabase.sqlite. This might be from an older Audiobookshelf server.') return res.status(500).send('Invalid backup file. Does not include absdatabase.sqlite. This might be from an older Audiobookshelf server.')
@ -266,6 +316,70 @@ class BackupManager {
SocketAuthority.emitter('backup_applied') SocketAuthority.emitter('backup_applied')
} }
async requestApplyPostgresBackup(apiCacheManager, backup, zip, entries, res) {
const databaseEntryName = this.getBackupEntryName('postgres')
if (!Object.keys(entries).includes(databaseEntryName)) {
Logger.error(`[BackupManager] Cannot apply Postgres backup ${backup.fullPath}`)
await zip.close()
return res.status(500).send(`Invalid backup file. Does not include ${databaseEntryName}.`)
}
const tempDumpPath = Path.join(global.ConfigPath, 'absdatabase-postgres-temp.dump')
let reconnected = false
let zipClosed = false
const closeZip = async () => {
if (zipClosed) return
zipClosed = true
await zip.close()
}
try {
await fs.remove(tempDumpPath)
await zip.extract(databaseEntryName, tempDumpPath)
if (!(await fs.pathExists(tempDumpPath))) {
await closeZip()
return res.status(500).send('Failed to extract Postgres database dump from backup')
}
await Database.disconnect()
await this.restorePostgresDb(tempDumpPath)
await fs.ensureDir(this.ItemsMetadataPath)
await zip.extract('metadata-items/', this.ItemsMetadataPath)
await fs.ensureDir(this.AuthorsMetadataPath)
await zip.extract('metadata-authors/', this.AuthorsMetadataPath)
await closeZip()
await Database.reconnect()
reconnected = true
await apiCacheManager.reset()
await CacheManager.purgeAll()
res.sendStatus(200)
SocketAuthority.emitter('backup_applied')
} catch (error) {
Logger.error(`[BackupManager] Failed to apply Postgres backup`, error)
try {
await closeZip()
} catch (closeError) {
Logger.error(`[BackupManager] Failed to close Postgres backup archive`, closeError)
}
if (!reconnected) {
try {
await Database.reconnect()
} catch (reconnectError) {
Logger.error(`[BackupManager] Failed to reconnect after Postgres backup apply`, reconnectError)
}
}
return res.status(500).send(`Failed to apply Postgres backup: ${error?.message || 'Unknown Error'}`)
} finally {
await fs.remove(tempDumpPath)
}
}
async loadBackups() { async loadBackups() {
try { try {
const filesInDir = await fs.readdir(this.backupPath) const filesInDir = await fs.readdir(this.backupPath)
@ -276,7 +390,7 @@ class BackupManager {
const fullFilePath = Path.join(this.backupPath, filename) const fullFilePath = Path.join(this.backupPath, filename)
let zip = null let zip = null
let data = null let backup = null
try { try {
zip = new StreamZip.async({ file: fullFilePath }) zip = new StreamZip.async({ file: fullFilePath })
const entries = await zip.entries() const entries = await zip.entries()
@ -293,17 +407,24 @@ class BackupManager {
continue continue
} }
data = await zip.entryData('details') const data = await zip.entryData('details')
const details = data.toString('utf8').split('\n')
backup = new Backup({ details, fullPath: fullFilePath })
const backupDialect = this.getBackupDialect(backup)
const databaseEntryName = this.getBackupEntryName(backupDialect)
if (!backupDialect || !entries[databaseEntryName]) {
Logger.error(`[BackupManager] Unsupported database backup format found "${backup.filename}"`)
await zip.close().catch(() => {})
continue
}
} catch (error) { } catch (error) {
Logger.error(`[BackupManager] Failed to unzip backup "${fullFilePath}"`, error) Logger.error(`[BackupManager] Failed to unzip backup "${fullFilePath}"`, error)
if (zip) await zip.close().catch(() => {}) if (zip) await zip.close().catch(() => {})
continue continue
} }
const details = data.toString('utf8').split('\n')
const backup = new Backup({ details, fullPath: fullFilePath })
if (!backup.serverVersion) { if (!backup.serverVersion) {
// Backups before v2 // Backups before v2
Logger.error(`[BackupManager] Old unsupported backup was found "${backup.filename}"`) Logger.error(`[BackupManager] Old unsupported backup was found "${backup.filename}"`)
@ -333,33 +454,34 @@ class BackupManager {
async runBackup() { async runBackup() {
// Check if Metadata Path is inside Config Path (otherwise there will be an infinite loop as the archiver tries to zip itself) // Check if Metadata Path is inside Config Path (otherwise there will be an infinite loop as the archiver tries to zip itself)
Logger.info(`[BackupManager] Running Backup`) Logger.info(`[BackupManager] Running Backup`)
const databaseBackupConfig = this.databaseBackupConfig
const newBackup = new Backup() const newBackup = new Backup()
newBackup.setData(this.backupPath) newBackup.setData(this.backupPath, databaseBackupConfig.dialect)
await fs.ensureDir(this.AuthorsMetadataPath) await fs.ensureDir(this.AuthorsMetadataPath)
// Create backup sqlite file // Create a database dump
const sqliteBackupPath = await this.backupSqliteDb(newBackup).catch((error) => { const databaseBackupPath = await this.backupDatabase(newBackup).catch((error) => {
Logger.error(`[BackupManager] Failed to backup sqlite db`, error) Logger.error(`[BackupManager] Failed to backup ${databaseBackupConfig.dialect} database`, error)
const errorMsg = error?.message || error || 'Unknown Error' const errorMsg = error?.message || error || 'Unknown Error'
NotificationManager.onBackupFailed(errorMsg) NotificationManager.onBackupFailed(errorMsg)
return false return false
}) })
if (!sqliteBackupPath) { if (!databaseBackupPath) {
return false return false
} }
// Zip sqlite file, /metadata/items, and /metadata/authors folders // Zip database dump, /metadata/items, and /metadata/authors folders
const zipResult = await this.zipBackup(sqliteBackupPath, newBackup).catch((error) => { const zipResult = await this.zipBackup(databaseBackupPath, newBackup, databaseBackupConfig.entryName).catch((error) => {
Logger.error(`[BackupManager] Backup Failed ${error}`) Logger.error(`[BackupManager] Backup Failed ${error}`)
const errorMsg = error?.message || error || 'Unknown Error' const errorMsg = error?.message || error || 'Unknown Error'
NotificationManager.onBackupFailed(errorMsg) NotificationManager.onBackupFailed(errorMsg)
return false return false
}) })
// Remove sqlite backup // Remove temporary database dump
await fs.remove(sqliteBackupPath) await fs.remove(databaseBackupPath)
if (!zipResult) return false if (!zipResult) return false
@ -390,6 +512,10 @@ class BackupManager {
return true return true
} }
backupDatabase(backup) {
return this.databaseBackupConfig.dialect === 'postgres' ? this.backupPostgresDb(backup) : this.backupSqliteDb(backup)
}
async removeBackup(backup) { async removeBackup(backup) {
try { try {
Logger.debug(`[BackupManager] Removing Backup "${backup.fullPath}"`) Logger.debug(`[BackupManager] Removing Backup "${backup.fullPath}"`)
@ -406,29 +532,147 @@ class BackupManager {
* @param {Backup} backup * @param {Backup} backup
*/ */
backupSqliteDb(backup) { backupSqliteDb(backup) {
const db = new sqlite3.Database(Database.dbPath)
const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.sqlite`) const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.sqlite`)
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
const backup = db.backup(dbFilePath) let db
backup.step(-1) let sqliteBackup
backup.finish() let settled = false
// Max time ~2 mins const finish = (error, result) => {
for (let i = 0; i < 240; i++) { if (settled) return
if (backup.completed) { settled = true
return resolve(dbFilePath) if (db) {
} else if (backup.failed) { db.close(() => {
return reject(backup.message || 'Unknown failure reason') if (error) reject(error)
else resolve(result)
})
} else if (error) {
reject(error)
} else {
resolve(result)
} }
await new Promise((r) => setTimeout(r, 500))
} }
Logger.error(`[BackupManager] Backup sqlite timed out`) const pollBackup = async () => {
reject('Backup timed out') // Max time ~2 mins
for (let i = 0; i < 240; i++) {
if (sqliteBackup.completed) {
return finish(null, dbFilePath)
} else if (sqliteBackup.failed) {
return finish(sqliteBackup.message || 'Unknown failure reason')
}
await new Promise((r) => setTimeout(r, 500))
}
Logger.error(`[BackupManager] Backup sqlite timed out`)
finish('Backup timed out')
}
const startBackup = () => {
try {
sqliteBackup = db.backup(dbFilePath)
sqliteBackup.step(-1)
sqliteBackup.finish()
pollBackup().catch(finish)
} catch (error) {
finish(error)
}
}
db = new sqlite3.Database(Database.dbPath, (error) => {
if (error) return finish(error)
startBackup()
})
db.on('error', finish)
}) })
} }
zipBackup(sqliteBackupPath, backup) { /**
* Build pg_dump/pg_restore connection arguments from DATABASE_URL without
* exposing credentials in argv. execFile error messages and the host process
* list include argv, so the password is passed via PGPASSWORD env instead.
*/
getPostgresConnection() {
let dbUrl
try {
dbUrl = new URL(Database.dbPath)
} catch (error) {
throw new Error('DATABASE_URL must be a valid postgres connection URI to run backups')
}
const args = ['--host', dbUrl.hostname, '--dbname', decodeURIComponent(dbUrl.pathname.replace(/^\//, ''))]
if (dbUrl.port) args.push('--port', dbUrl.port)
if (dbUrl.username) args.push('--username', decodeURIComponent(dbUrl.username))
// Redact both the percent-encoded and decoded password from any error output
const decodedPassword = dbUrl.password ? decodeURIComponent(dbUrl.password) : null
const secrets = dbUrl.password ? [dbUrl.password, decodedPassword] : []
const env = decodedPassword ? { ...process.env, PGPASSWORD: decodedPassword } : process.env
return { args, env, secrets }
}
backupPostgresDb(backup) {
const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.postgres.dump`)
return this.runPostgresCommand('pg_dump', [
'--format=custom',
'--no-owner',
'--no-acl',
'--file',
dbFilePath
])
.then(() => dbFilePath)
.catch(async (error) => {
await fs.remove(dbFilePath)
throw error
})
}
restorePostgresDb(dbFilePath) {
return this.runPostgresCommand('pg_restore', [
'--clean',
'--if-exists',
'--exit-on-error',
'--single-transaction',
'--no-owner',
'--no-acl',
dbFilePath
])
}
runPostgresCommand(command, args) {
return new Promise((resolve, reject) => {
let connection
try {
connection = this.getPostgresConnection()
} catch (error) {
return reject(error)
}
const redact = (text) => {
if (typeof text !== 'string') return text
return connection.secrets.reduce((redacted, secret) => redacted.split(secret).join('***'), text)
}
const options = {
maxBuffer: 10 * 1024 * 1024,
// Kill after 30 mins (e.g. lock waits) - a killed restore rolls back via --single-transaction
timeout: 30 * 60 * 1000,
env: connection.env
}
childProcess.execFile(command, [...args, ...connection.args], options, (error, stdout, stderr) => {
if (error) {
error.message = redact(error.message)
if (error.cmd) error.cmd = redact(error.cmd)
error.stderr = redact(stderr)
return reject(error)
}
resolve({ stdout, stderr })
})
})
}
zipBackup(databaseBackupPath, backup, databaseEntryName = 'absdatabase.sqlite') {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
// create a file to stream archive data to // create a file to stream archive data to
const output = fs.createWriteStream(backup.fullPath) const output = fs.createWriteStream(backup.fullPath)
@ -492,7 +736,7 @@ class BackupManager {
// pipe archive data to the file // pipe archive data to the file
archive.pipe(output) archive.pipe(output)
archive.file(sqliteBackupPath, { name: 'absdatabase.sqlite' }) archive.file(databaseBackupPath, { name: databaseEntryName })
archive.directory(this.ItemsMetadataPath, 'metadata-items') archive.directory(this.ItemsMetadataPath, 'metadata-items')
archive.directory(this.AuthorsMetadataPath, 'metadata-authors') archive.directory(this.AuthorsMetadataPath, 'metadata-authors')

View file

@ -94,20 +94,26 @@ class MigrationManager {
// Only proceed with migration if there are migrations to run // Only proceed with migration if there are migrations to run
if (migrationsToRun.length > 0) { if (migrationsToRun.length > 0) {
const dialect = typeof this.sequelize.getDialect === 'function' ? this.sequelize.getDialect() : 'sqlite'
const isSqlite = !dialect || dialect === 'sqlite'
const originalDbPath = path.join(this.configPath, 'absdatabase.sqlite') const originalDbPath = path.join(this.configPath, 'absdatabase.sqlite')
const backupDbPath = path.join(this.configPath, 'absdatabase.backup.sqlite') const backupDbPath = path.join(this.configPath, 'absdatabase.backup.sqlite')
try { try {
Logger.info(`[MigrationManager] Migrating database ${migrationDirection} to version ${this.serverVersion}`) Logger.info(`[MigrationManager] Migrating database ${migrationDirection} to version ${this.serverVersion}`)
Logger.info(`[MigrationManager] Migrations to run: ${migrationsToRun.join(', ')}`) Logger.info(`[MigrationManager] Migrations to run: ${migrationsToRun.join(', ')}`)
// Create a backup copy of the SQLite database before starting migrations if (isSqlite) {
await fs.copy(originalDbPath, backupDbPath) // Create a backup copy of the SQLite database before starting migrations
Logger.info('Created a backup of the original database.') await fs.copy(originalDbPath, backupDbPath)
Logger.info('Created a backup of the original database.')
}
// Run migrations // Run migrations
await this.umzug[migrationDirection]({ migrations: migrationsToRun, rerun: 'ALLOW' }) await this.umzug[migrationDirection]({ migrations: migrationsToRun, rerun: 'ALLOW' })
// Clean up the backup if (isSqlite) {
await fs.remove(backupDbPath) // Clean up the backup
await fs.remove(backupDbPath)
}
Logger.info('[MigrationManager] Migrations successfully applied to the original database.') Logger.info('[MigrationManager] Migrations successfully applied to the original database.')
} catch (error) { } catch (error) {
@ -115,13 +121,15 @@ class MigrationManager {
await this.sequelize.close() await this.sequelize.close()
// Step 3: If migration fails, save the failed original and restore the backup if (isSqlite) {
const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite') // Step 3: If migration fails, save the failed original and restore the backup
await fs.move(originalDbPath, failedDbPath, { overwrite: true }) const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite')
Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.') await fs.move(originalDbPath, failedDbPath, { overwrite: true })
Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.')
await fs.move(backupDbPath, originalDbPath, { overwrite: true }) await fs.move(backupDbPath, originalDbPath, { overwrite: true })
Logger.info('[MigrationManager] Restored the original database from the backup.') Logger.info('[MigrationManager] Restored the original database from the backup.')
}
Logger.info('[MigrationManager] Migration failed. Exiting Audiobookshelf with code 1.') Logger.info('[MigrationManager] Migration failed. Exiting Audiobookshelf with code 1.')
process.exit(1) process.exit(1)
@ -191,29 +199,27 @@ class MigrationManager {
} }
async fetchVersionsFromDatabase() { async fetchVersionsFromDatabase() {
const migrationsMetaTable = MigrationManager.MIGRATIONS_META_TABLE
await this.checkOrCreateMigrationsMetaTable() await this.checkOrCreateMigrationsMetaTable()
const [{ version }] = await this.sequelize.query("SELECT value as version FROM :migrationsMeta WHERE key = 'version'", { const [versionRow] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, {
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
type: Sequelize.QueryTypes.SELECT type: Sequelize.QueryTypes.SELECT
}) })
this.databaseVersion = version this.databaseVersion = versionRow?.version
const [{ maxVersion }] = await this.sequelize.query("SELECT value as maxVersion FROM :migrationsMeta WHERE key = 'maxVersion'", { const [maxVersionRow] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, {
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
type: Sequelize.QueryTypes.SELECT type: Sequelize.QueryTypes.SELECT
}) })
this.maxVersion = maxVersion this.maxVersion = maxVersionRow?.maxVersion || maxVersionRow?.maxversion
} }
async checkOrCreateMigrationsMetaTable() { async checkOrCreateMigrationsMetaTable() {
const queryInterface = this.sequelize.getQueryInterface() const queryInterface = this.sequelize.getQueryInterface()
let migrationsMetaTableExists = await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE) let migrationsMetaTableExists = await this.tableExists(MigrationManager.MIGRATIONS_META_TABLE)
// If the table exists, check that the `version` and `maxVersion` rows exist // If the table exists, check that the `version` and `maxVersion` rows exist
if (migrationsMetaTableExists) { if (migrationsMetaTableExists) {
const [{ count }] = await this.sequelize.query("SELECT COUNT(*) as count FROM :migrationsMeta WHERE key IN ('version', 'maxVersion')", { const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM ${MigrationManager.MIGRATIONS_META_TABLE} WHERE key IN ('version', 'maxVersion')`, {
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
type: Sequelize.QueryTypes.SELECT type: Sequelize.QueryTypes.SELECT
}) })
if (count < 2) { if (count < 2) {
@ -241,14 +247,28 @@ class MigrationManager {
allowNull: false allowNull: false
} }
}) })
await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", { await this.sequelize.query(`INSERT INTO ${MigrationManager.MIGRATIONS_META_TABLE} (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, {
replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0' },
type: Sequelize.QueryTypes.INSERT type: Sequelize.QueryTypes.INSERT
}) })
Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`) Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`)
} }
} }
async tableExists(tableName) {
const queryInterface = this.sequelize.getQueryInterface()
if (typeof queryInterface.tableExists === 'function') {
return queryInterface.tableExists(tableName)
}
const tables = await queryInterface.showAllTables()
return tables.some((table) => {
if (typeof table === 'string') return table === tableName
if (table?.tableName) return table.tableName === tableName
return false
})
}
extractVersionFromTag(tag) { extractVersionFromTag(tag) {
if (!tag) return null if (!tag) return null
const versionMatch = tag.match(/^v?(\d+\.\d+\.\d+)/) const versionMatch = tag.match(/^v?(\d+\.\d+\.\d+)/)
@ -299,8 +319,8 @@ class MigrationManager {
async updateMaxVersion() { async updateMaxVersion() {
try { try {
await this.sequelize.query("UPDATE :migrationsMeta SET value = :maxVersion WHERE key = 'maxVersion'", { await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :maxVersion WHERE key = 'maxVersion'`, {
replacements: { maxVersion: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, replacements: { maxVersion: this.serverVersion },
type: Sequelize.QueryTypes.UPDATE type: Sequelize.QueryTypes.UPDATE
}) })
} catch (error) { } catch (error) {
@ -311,8 +331,8 @@ class MigrationManager {
async updateDatabaseVersion() { async updateDatabaseVersion() {
try { try {
await this.sequelize.query("UPDATE :migrationsMeta SET value = :version WHERE key = 'version'", { await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :version WHERE key = 'version'`, {
replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, replacements: { version: this.serverVersion },
type: Sequelize.QueryTypes.UPDATE type: Sequelize.QueryTypes.UPDATE
}) })
} catch (error) { } catch (error) {

View file

@ -19,12 +19,21 @@ async function up({ context: { queryInterface, logger } }) {
logger.info('[2.15.0 migration] UPGRADE BEGIN: 2.15.0-series-column-unique ') logger.info('[2.15.0 migration] UPGRADE BEGIN: 2.15.0-series-column-unique ')
// Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0 // Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0
logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') if (queryInterface.sequelize.getDialect() === 'sqlite') {
await queryInterface.sequelize.query('REINDEX NOCASE;') logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues')
await queryInterface.sequelize.query('REINDEX NOCASE;')
} else {
logger.info('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect')
}
// Check if the unique index already exists // Check if the unique index already exists
const seriesIndexes = await queryInterface.showIndex('Series') const seriesIndexes = await queryInterface.showIndex('Series')
if (seriesIndexes.some((index) => index.name === 'unique_series_name_per_library')) { if (
seriesIndexes.some((index) => {
const indexName = index?.name || index?.indexName || ''
return String(indexName).toLowerCase() === 'unique_series_name_per_library'
})
) {
logger.info('[2.15.0 migration] Unique index on Series.name and Series.libraryId already exists') logger.info('[2.15.0 migration] Unique index on Series.name and Series.libraryId already exists')
logger.info('[2.15.0 migration] UPGRADE END: 2.15.0-series-column-unique ') logger.info('[2.15.0 migration] UPGRADE END: 2.15.0-series-column-unique ')
return return
@ -181,11 +190,20 @@ async function up({ context: { queryInterface, logger } }) {
logger.info(`[2.15.0 migration] Deduplication complete`) logger.info(`[2.15.0 migration] Deduplication complete`)
// Create a unique index based on the name and library ID for the `Series` table // Create a unique index based on the name and library ID for the `Series` table
await queryInterface.addIndex('Series', ['name', 'libraryId'], { try {
unique: true, await queryInterface.addIndex('Series', ['name', 'libraryId'], {
name: 'unique_series_name_per_library' unique: true,
}) name: 'unique_series_name_per_library'
logger.info('[2.15.0 migration] Added unique index on Series.name and Series.libraryId') })
logger.info('[2.15.0 migration] Added unique index on Series.name and Series.libraryId')
} catch (error) {
const alreadyExists =
(error?.name === 'SequelizeDatabaseError' && /already exists/i.test(error?.message || '')) ||
error?.original?.code === '42P07'
if (!alreadyExists) throw error
logger.info('[2.15.0 migration] Unique index on Series.name and Series.libraryId already exists')
}
logger.info('[2.15.0 migration] UPGRADE END: 2.15.0-series-column-unique ') logger.info('[2.15.0 migration] UPGRADE END: 2.15.0-series-column-unique ')
} }

View file

@ -17,6 +17,12 @@ async function up({ context: { queryInterface, logger } }) {
// Upwards migration script // Upwards migration script
logger.info('[2.15.1 migration] UPGRADE BEGIN: 2.15.1-reindex-nocase ') logger.info('[2.15.1 migration] UPGRADE BEGIN: 2.15.1-reindex-nocase ')
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect')
logger.info('[2.15.1 migration] UPGRADE END: 2.15.1-reindex-nocase ')
return
}
// Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0 // Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0
logger.info('[2.15.1 migration] Reindexing NOCASE indices to fix potential hidden corruption issues') logger.info('[2.15.1 migration] Reindexing NOCASE indices to fix potential hidden corruption issues')
await queryInterface.sequelize.query('REINDEX NOCASE;') await queryInterface.sequelize.query('REINDEX NOCASE;')

View file

@ -20,10 +20,8 @@ async function up({ context: { queryInterface, logger } }) {
// Create index for bookAuthors // Create index for bookAuthors
logger.info('[2.15.2 migration] Creating index for bookAuthors') logger.info('[2.15.2 migration] Creating index for bookAuthors')
const bookAuthorsIndexes = await queryInterface.showIndex('bookAuthors') const bookAuthorsIndexes = await queryInterface.showIndex('bookAuthors')
if (!bookAuthorsIndexes.some((index) => index.name === 'bookAuthor_authorId')) { if (!hasIndex(bookAuthorsIndexes, 'bookAuthor_authorId')) {
await queryInterface.addIndex('bookAuthors', ['authorId'], { await addIndexIfMissing(queryInterface, 'bookAuthors', ['authorId'], 'bookAuthor_authorId', logger)
name: 'bookAuthor_authorId'
})
} else { } else {
logger.info('[2.15.2 migration] Index bookAuthor_authorId already exists') logger.info('[2.15.2 migration] Index bookAuthor_authorId already exists')
} }
@ -31,25 +29,21 @@ async function up({ context: { queryInterface, logger } }) {
// Create index for bookSeries // Create index for bookSeries
logger.info('[2.15.2 migration] Creating index for bookSeries') logger.info('[2.15.2 migration] Creating index for bookSeries')
const bookSeriesIndexes = await queryInterface.showIndex('bookSeries') const bookSeriesIndexes = await queryInterface.showIndex('bookSeries')
if (!bookSeriesIndexes.some((index) => index.name === 'bookSeries_seriesId')) { if (!hasIndex(bookSeriesIndexes, 'bookSeries_seriesId')) {
await queryInterface.addIndex('bookSeries', ['seriesId'], { await addIndexIfMissing(queryInterface, 'bookSeries', ['seriesId'], 'bookSeries_seriesId', logger)
name: 'bookSeries_seriesId'
})
} else { } else {
logger.info('[2.15.2 migration] Index bookSeries_seriesId already exists') logger.info('[2.15.2 migration] Index bookSeries_seriesId already exists')
} }
// Delete existing podcastEpisode index // Delete existing podcastEpisode index
logger.info('[2.15.2 migration] Deleting existing podcastEpisode index') logger.info('[2.15.2 migration] Deleting existing podcastEpisode index')
await queryInterface.removeIndex('podcastEpisodes', 'podcast_episodes_created_at') await removeIndexIfExists(queryInterface, 'podcastEpisodes', 'podcast_episodes_created_at', logger)
// Create index for podcastEpisode and createdAt // Create index for podcastEpisode and createdAt
logger.info('[2.15.2 migration] Creating index for podcastEpisode and createdAt') logger.info('[2.15.2 migration] Creating index for podcastEpisode and createdAt')
const podcastEpisodesIndexes = await queryInterface.showIndex('podcastEpisodes') const podcastEpisodesIndexes = await queryInterface.showIndex('podcastEpisodes')
if (!podcastEpisodesIndexes.some((index) => index.name === 'podcastEpisode_createdAt_podcastId')) { if (!hasIndex(podcastEpisodesIndexes, 'podcastEpisode_createdAt_podcastId')) {
await queryInterface.addIndex('podcastEpisodes', ['createdAt', 'podcastId'], { await addIndexIfMissing(queryInterface, 'podcastEpisodes', ['createdAt', 'podcastId'], 'podcastEpisode_createdAt_podcastId', logger)
name: 'podcastEpisode_createdAt_podcastId'
})
} else { } else {
logger.info('[2.15.2 migration] Index podcastEpisode_createdAt_podcastId already exists') logger.info('[2.15.2 migration] Index podcastEpisode_createdAt_podcastId already exists')
} }
@ -78,7 +72,7 @@ async function down({ context: { queryInterface, logger } }) {
// Delete existing podcastEpisode index // Delete existing podcastEpisode index
logger.info('[2.15.2 migration] Deleting existing podcastEpisode index') logger.info('[2.15.2 migration] Deleting existing podcastEpisode index')
await queryInterface.removeIndex('podcastEpisodes', 'podcastEpisode_createdAt_podcastId') await removeIndexIfExists(queryInterface, 'podcastEpisodes', 'podcastEpisode_createdAt_podcastId', logger)
// Create index for podcastEpisode and createdAt // Create index for podcastEpisode and createdAt
logger.info('[2.15.2 migration] Creating original index for podcastEpisode createdAt') logger.info('[2.15.2 migration] Creating original index for podcastEpisode createdAt')
@ -91,3 +85,33 @@ async function down({ context: { queryInterface, logger } }) {
} }
module.exports = { up, down } module.exports = { up, down }
async function removeIndexIfExists(queryInterface, tableName, indexName, logger) {
const indexes = await queryInterface.showIndex(tableName)
const hasIndexWithName = hasIndex(indexes, indexName)
if (!hasIndexWithName) {
logger.info(`[2.15.2 migration] Index ${indexName} does not exist, skipping removeIndex`)
return
}
await queryInterface.removeIndex(tableName, indexName)
}
function hasIndex(indexes, indexName) {
const expected = String(indexName || '').toLowerCase()
return indexes.some((index) => String(index?.name || index?.indexName || '').toLowerCase() === expected)
}
async function addIndexIfMissing(queryInterface, tableName, fields, indexName, logger) {
try {
await queryInterface.addIndex(tableName, fields, { name: indexName })
} catch (error) {
const alreadyExists =
(error?.name === 'SequelizeDatabaseError' && /already exists/i.test(error?.message || '')) ||
error?.original?.code === '42P07'
if (!alreadyExists) throw error
logger.info(`[2.15.2 migration] Index ${indexName} already exists`)
}
}

View file

@ -18,6 +18,12 @@ async function up({ context: { queryInterface, logger } }) {
// Upwards migration script // Upwards migration script
logger.info('[2.17.3 migration] UPGRADE BEGIN: 2.17.3-fk-constraints') logger.info('[2.17.3 migration] UPGRADE BEGIN: 2.17.3-fk-constraints')
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect')
logger.info('[2.17.3 migration] UPGRADE END: 2.17.3-fk-constraints')
return
}
const execQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize) const execQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize)
// Disable foreign key constraints for the next sequence of operations // Disable foreign key constraints for the next sequence of operations

View file

@ -56,15 +56,23 @@ async function down({ context: { queryInterface, logger } }) {
} }
async function getServerSettings(queryInterface, logger) { async function getServerSettings(queryInterface, logger) {
const result = await queryInterface.sequelize.query('SELECT value FROM settings WHERE key = "server-settings";') const result = await queryInterface.sequelize.query('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
if (!result[0].length) { if (!result[0].length) {
logger.error('[2.17.4 migration] Server settings not found') logger.error('[2.17.4 migration] Server settings not found')
throw new Error('Server settings not found') throw new Error('Server settings not found')
} }
const settingsValue = result[0][0].value
if (settingsValue && typeof settingsValue === 'object') {
return settingsValue
}
let serverSettings = null let serverSettings = null
try { try {
serverSettings = JSON.parse(result[0][0].value) serverSettings = JSON.parse(settingsValue)
} catch (error) { } catch (error) {
logger.error('[2.17.4 migration] Error parsing server settings:', error) logger.error('[2.17.4 migration] Error parsing server settings:', error)
throw error throw error
@ -74,9 +82,10 @@ async function getServerSettings(queryInterface, logger) {
} }
async function updateServerSettings(queryInterface, logger, serverSettings) { async function updateServerSettings(queryInterface, logger, serverSettings) {
await queryInterface.sequelize.query('UPDATE settings SET value = :value WHERE key = "server-settings";', { await queryInterface.sequelize.query('UPDATE settings SET value = :value WHERE key = :settingsKey;', {
replacements: { replacements: {
value: JSON.stringify(serverSettings) value: JSON.stringify(serverSettings),
settingsKey: 'server-settings'
} }
}) })
} }

View file

@ -25,6 +25,12 @@ async function up({ context: { queryInterface, logger } }) {
// Upwards migration script // Upwards migration script
logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`)
logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
return
}
await addColumn(queryInterface, logger, 'libraryItems', 'title', { type: queryInterface.sequelize.Sequelize.STRING, allowNull: true }) await addColumn(queryInterface, logger, 'libraryItems', 'title', { type: queryInterface.sequelize.Sequelize.STRING, allowNull: true })
await copyColumn(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId') await copyColumn(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId')
await addTrigger(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId') await addTrigger(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId')
@ -51,6 +57,12 @@ async function down({ context: { queryInterface, logger } }) {
// Downward migration script // Downward migration script
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`)
logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
return
}
await removeIndex(queryInterface, logger, 'libraryItems', ['libraryId', 'mediaType', 'title']) await removeIndex(queryInterface, logger, 'libraryItems', ['libraryId', 'mediaType', 'title'])
await removeTrigger(queryInterface, logger, 'libraryItems', 'title') await removeTrigger(queryInterface, logger, 'libraryItems', 'title')
await removeColumn(queryInterface, logger, 'libraryItems', 'title') await removeColumn(queryInterface, logger, 'libraryItems', 'title')

View file

@ -26,6 +26,12 @@ async function up({ context: { queryInterface, logger } }) {
// Upwards migration script // Upwards migration script
logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`)
logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
return
}
// Add numEpisodes column to podcasts table // Add numEpisodes column to podcasts table
await addColumn(queryInterface, logger, 'podcasts', 'numEpisodes', { type: queryInterface.sequelize.Sequelize.INTEGER, allowNull: false, defaultValue: 0 }) await addColumn(queryInterface, logger, 'podcasts', 'numEpisodes', { type: queryInterface.sequelize.Sequelize.INTEGER, allowNull: false, defaultValue: 0 })
@ -60,6 +66,12 @@ async function down({ context: { queryInterface, logger } }) {
// Downward migration script // Downward migration script
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`)
logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
return
}
// Remove triggers from libraryItems // Remove triggers from libraryItems
await removeTrigger(queryInterface, logger, 'podcasts', 'title', 'libraryItems', 'title') await removeTrigger(queryInterface, logger, 'podcasts', 'title', 'libraryItems', 'title')
await removeTrigger(queryInterface, logger, 'podcasts', 'titleIgnorePrefix', 'libraryItems', 'titleIgnorePrefix') await removeTrigger(queryInterface, logger, 'podcasts', 'titleIgnorePrefix', 'libraryItems', 'titleIgnorePrefix')

View file

@ -39,6 +39,11 @@ const authorsJoin = `${authors} JOIN ${bookAuthors} ON ${authors}.id = ${bookAut
* @returns {Promise<void>} - A promise that resolves when the migration is complete. * @returns {Promise<void>} - A promise that resolves when the migration is complete.
*/ */
async function up({ context: { queryInterface, logger } }) { async function up({ context: { queryInterface, logger } }) {
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`)
return
}
const helper = new MigrationHelper(queryInterface, logger) const helper = new MigrationHelper(queryInterface, logger)
// Upwards migration script // Upwards migration script
@ -72,6 +77,11 @@ async function up({ context: { queryInterface, logger } }) {
* @returns {Promise<void>} - A promise that resolves when the migration is complete. * @returns {Promise<void>} - A promise that resolves when the migration is complete.
*/ */
async function down({ context: { queryInterface, logger } }) { async function down({ context: { queryInterface, logger } }) {
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`)
return
}
// Downward migration script // Downward migration script
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)

View file

@ -22,8 +22,10 @@ async function up({ context: { queryInterface, logger } }) {
if (await queryInterface.tableExists('sessions')) { if (await queryInterface.tableExists('sessions')) {
const tableDescription = await queryInterface.describeTable('sessions') const tableDescription = await queryInterface.describeTable('sessions')
// Postgres folds unquoted identifiers to lowercase, so match column names case-insensitively
const hasColumn = (name) => Object.keys(tableDescription).some((column) => column.toLowerCase() === name.toLowerCase())
if (!tableDescription.lastRefreshToken) { if (!hasColumn('lastRefreshToken')) {
logger.info(`${loggerPrefix} Adding lastRefreshToken column to sessions table`) logger.info(`${loggerPrefix} Adding lastRefreshToken column to sessions table`)
await queryInterface.addColumn('sessions', 'lastRefreshToken', { await queryInterface.addColumn('sessions', 'lastRefreshToken', {
type: queryInterface.sequelize.Sequelize.DataTypes.STRING, type: queryInterface.sequelize.Sequelize.DataTypes.STRING,
@ -33,7 +35,7 @@ async function up({ context: { queryInterface, logger } }) {
logger.info(`${loggerPrefix} lastRefreshToken column already exists in sessions table`) logger.info(`${loggerPrefix} lastRefreshToken column already exists in sessions table`)
} }
if (!tableDescription.lastRefreshTokenExpiresAt) { if (!hasColumn('lastRefreshTokenExpiresAt')) {
logger.info(`${loggerPrefix} Adding lastRefreshTokenExpiresAt column to sessions table`) logger.info(`${loggerPrefix} Adding lastRefreshTokenExpiresAt column to sessions table`)
await queryInterface.addColumn('sessions', 'lastRefreshTokenExpiresAt', { await queryInterface.addColumn('sessions', 'lastRefreshTokenExpiresAt', {
type: queryInterface.sequelize.Sequelize.DataTypes.DATE, type: queryInterface.sequelize.Sequelize.DataTypes.DATE,
@ -60,15 +62,17 @@ async function down({ context: { queryInterface, logger } }) {
if (await queryInterface.tableExists('sessions')) { if (await queryInterface.tableExists('sessions')) {
const tableDescription = await queryInterface.describeTable('sessions') const tableDescription = await queryInterface.describeTable('sessions')
// Postgres folds unquoted identifiers to lowercase, so match column names case-insensitively
const hasColumn = (name) => Object.keys(tableDescription).some((column) => column.toLowerCase() === name.toLowerCase())
if (tableDescription.lastRefreshToken) { if (hasColumn('lastRefreshToken')) {
logger.info(`${loggerPrefix} Removing lastRefreshToken column from sessions table`) logger.info(`${loggerPrefix} Removing lastRefreshToken column from sessions table`)
await queryInterface.removeColumn('sessions', 'lastRefreshToken') await queryInterface.removeColumn('sessions', 'lastRefreshToken')
} else { } else {
logger.info(`${loggerPrefix} lastRefreshToken column does not exist in sessions table`) logger.info(`${loggerPrefix} lastRefreshToken column does not exist in sessions table`)
} }
if (tableDescription.lastRefreshTokenExpiresAt) { if (hasColumn('lastRefreshTokenExpiresAt')) {
logger.info(`${loggerPrefix} Removing lastRefreshTokenExpiresAt column from sessions table`) logger.info(`${loggerPrefix} Removing lastRefreshTokenExpiresAt column from sessions table`)
await queryInterface.removeColumn('sessions', 'lastRefreshTokenExpiresAt') await queryInterface.removeColumn('sessions', 'lastRefreshTokenExpiresAt')
} else { } else {

View file

@ -129,6 +129,8 @@ class Author extends Model {
* @param {import('../Database').sequelize} sequelize * @param {import('../Database').sequelize} sequelize
*/ */
static init(sequelize) { static init(sequelize) {
const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' }
super.init( super.init(
{ {
id: { id: {
@ -147,12 +149,7 @@ class Author extends Model {
modelName: 'author', modelName: 'author',
indexes: [ indexes: [
{ {
fields: [ fields: [nameIndexField]
{
name: 'name',
collate: 'NOCASE'
}
]
}, },
// { // {
// fields: [{ // fields: [{

View file

@ -138,6 +138,8 @@ class Book extends Model {
* @param {import('../Database').sequelize} sequelize * @param {import('../Database').sequelize} sequelize
*/ */
static init(sequelize) { static init(sequelize) {
const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' }
super.init( super.init(
{ {
id: { id: {
@ -147,7 +149,7 @@ class Book extends Model {
}, },
title: DataTypes.STRING, title: DataTypes.STRING,
titleIgnorePrefix: DataTypes.STRING, titleIgnorePrefix: DataTypes.STRING,
subtitle: DataTypes.STRING, subtitle: DataTypes.TEXT,
publishedYear: DataTypes.STRING, publishedYear: DataTypes.STRING,
publishedDate: DataTypes.STRING, publishedDate: DataTypes.STRING,
publisher: DataTypes.STRING, publisher: DataTypes.STRING,
@ -172,12 +174,7 @@ class Book extends Model {
modelName: 'book', modelName: 'book',
indexes: [ indexes: [
{ {
fields: [ fields: [titleIndexField]
{
name: 'title',
collate: 'NOCASE'
}
]
}, },
// { // {
// fields: [{ // fields: [{

View file

@ -720,6 +720,11 @@ class LibraryItem extends Model {
* @param {import('../Database').sequelize} sequelize * @param {import('../Database').sequelize} sequelize
*/ */
static init(sequelize) { static init(sequelize) {
const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' }
const titleIgnorePrefixIndexField = sequelize.getDialect() === 'postgres' ? 'titleIgnorePrefix' : { name: 'titleIgnorePrefix', collate: 'NOCASE' }
const authorNamesFirstLastIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesFirstLast' : { name: 'authorNamesFirstLast', collate: 'NOCASE' }
const authorNamesLastFirstIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesLastFirst' : { name: 'authorNamesLastFirst', collate: 'NOCASE' }
super.init( super.init(
{ {
id: { id: {
@ -768,16 +773,16 @@ class LibraryItem extends Model {
fields: ['libraryId', 'mediaType', 'createdAt'] fields: ['libraryId', 'mediaType', 'createdAt']
}, },
{ {
fields: ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }] fields: ['libraryId', 'mediaType', titleIndexField]
}, },
{ {
fields: ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }] fields: ['libraryId', 'mediaType', titleIgnorePrefixIndexField]
}, },
{ {
fields: ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }] fields: ['libraryId', 'mediaType', authorNamesFirstLastIndexField]
}, },
{ {
fields: ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }] fields: ['libraryId', 'mediaType', authorNamesLastFirstIndexField]
}, },
{ {
fields: ['libraryId', 'mediaId', 'mediaType'] fields: ['libraryId', 'mediaId', 'mediaType']

View file

@ -257,7 +257,13 @@ class MediaProgress extends Model {
const escapedDate = this.sequelize.escape(new Date(progressPayload.lastUpdate)) const escapedDate = this.sequelize.escape(new Date(progressPayload.lastUpdate))
Logger.info(`[MediaProgress] Manually setting updatedAt to ${escapedDate} (media item ${this.mediaItemId})`) Logger.info(`[MediaProgress] Manually setting updatedAt to ${escapedDate} (media item ${this.mediaItemId})`)
await this.sequelize.query(`UPDATE "mediaProgresses" SET "updatedAt" = ${escapedDate} WHERE "id" = '${this.id}'`) await this.constructor.update(
{ updatedAt: new Date(progressPayload.lastUpdate) },
{
where: { id: this.id },
silent: true
}
)
await this.reload() await this.reload()
} }

View file

@ -52,18 +52,54 @@ class PlaybackSession extends Model {
this.createdAt this.createdAt
} }
static async getOldPlaybackSessions(where = null) { static async getOldPlaybackSessions(where = null, options = {}) {
const {
limit = null,
offset = null,
order = [['updatedAt', 'DESC']],
includeDevice = true
} = options
const playbackSessions = await this.findAll({ const playbackSessions = await this.findAll({
where, where,
include: [ limit,
{ offset,
model: this.sequelize.models.device order,
} include: includeDevice
] ? [
{
model: this.sequelize.models.device
}
]
: undefined
}) })
return playbackSessions.map((session) => this.getOldPlaybackSession(session)) return playbackSessions.map((session) => this.getOldPlaybackSession(session))
} }
static countWithWhere(where = null) {
return this.count({ where })
}
static getPlaybackSessionsForStats(where = null) {
return this.findAll({
where,
order: [['updatedAt', 'DESC']],
attributes: [
'mediaItemId',
'mediaItemType',
'displayTitle',
'displayAuthor',
'timeListening',
'mediaMetadata',
'date',
'dayOfWeek',
'updatedAt',
'extraData'
],
raw: true
})
}
static async getById(sessionId) { static async getById(sessionId) {
const playbackSession = await this.findByPk(sessionId, { const playbackSession = await this.findByPk(sessionId, {
include: [ include: [
@ -190,7 +226,7 @@ class PlaybackSession extends Model {
currentTime: DataTypes.FLOAT, currentTime: DataTypes.FLOAT,
serverVersion: DataTypes.STRING, serverVersion: DataTypes.STRING,
coverPath: DataTypes.STRING, coverPath: DataTypes.STRING,
timeListening: DataTypes.INTEGER, timeListening: DataTypes.FLOAT,
mediaMetadata: DataTypes.JSON, mediaMetadata: DataTypes.JSON,
date: DataTypes.STRING, date: DataTypes.STRING,
dayOfWeek: DataTypes.STRING, dayOfWeek: DataTypes.STRING,
@ -237,18 +273,20 @@ class PlaybackSession extends Model {
if (!Array.isArray(findResult)) findResult = [findResult] if (!Array.isArray(findResult)) findResult = [findResult]
for (const instance of findResult) { for (const instance of findResult) {
const values = instance?.dataValues
if (instance.mediaItemType === 'book' && instance.book !== undefined) { if (instance.mediaItemType === 'book' && instance.book !== undefined) {
instance.mediaItem = instance.book instance.mediaItem = instance.book
instance.dataValues.mediaItem = instance.dataValues.book if (values) values.mediaItem = values.book
} else if (instance.mediaItemType === 'podcastEpisode' && instance.podcastEpisode !== undefined) { } else if (instance.mediaItemType === 'podcastEpisode' && instance.podcastEpisode !== undefined) {
instance.mediaItem = instance.podcastEpisode instance.mediaItem = instance.podcastEpisode
instance.dataValues.mediaItem = instance.dataValues.podcastEpisode if (values) values.mediaItem = values.podcastEpisode
} }
// To prevent mistakes: // To prevent mistakes:
delete instance.book delete instance.book
delete instance.dataValues.book if (values) delete values.book
delete instance.podcastEpisode delete instance.podcastEpisode
delete instance.dataValues.podcastEpisode if (values) delete values.podcastEpisode
} }
}) })
} }

View file

@ -1,6 +1,7 @@
const { DataTypes, Model, where, fn, col, literal } = require('sequelize') const { DataTypes, Model, where, fn, col, literal } = require('sequelize')
const { getTitlePrefixAtEnd, getTitleIgnorePrefix } = require('../utils/index') const { getTitlePrefixAtEnd, getTitleIgnorePrefix } = require('../utils/index')
const { safeTextToDoubleExpression } = require('../utils/sqlDialectHelpers')
class Series extends Model { class Series extends Model {
constructor(values, options) { constructor(values, options) {
@ -87,6 +88,8 @@ class Series extends Model {
* @param {import('../Database').sequelize} sequelize * @param {import('../Database').sequelize} sequelize
*/ */
static init(sequelize) { static init(sequelize) {
const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' }
super.init( super.init(
{ {
id: { id: {
@ -103,12 +106,7 @@ class Series extends Model {
modelName: 'series', modelName: 'series',
indexes: [ indexes: [
{ {
fields: [ fields: [nameIndexField]
{
name: 'name',
collate: 'NOCASE'
}
]
}, },
// { // {
// fields: [{ // fields: [{
@ -161,7 +159,7 @@ class Series extends Model {
} }
} }
], ],
order: [[literal('CAST(`bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]] order: [[literal(`${safeTextToDoubleExpression(this.sequelize.getDialect() === 'postgres' ? '"bookSeries"."sequence"' : '`bookSeries.sequence`', this.sequelize)} ASC NULLS LAST`)]]
}) })
} }

View file

@ -18,12 +18,16 @@ class UserCache {
} }
getByEmail(email) { getByEmail(email) {
const user = this.cache.find((u) => u.email === email) if (!email) return null
const normalizedEmail = email.toLowerCase()
const user = this.cache.find((u) => u.email && u.email.toLowerCase() === normalizedEmail)
return user return user
} }
getByUsername(username) { getByUsername(username) {
const user = this.cache.find((u) => u.username === username) if (!username) return null
const normalizedUsername = username.toLowerCase()
const user = this.cache.find((u) => u.username && u.username.toLowerCase() === normalizedUsername)
return user return user
} }
@ -348,11 +352,13 @@ class User extends Model {
static async getUserByUsername(username) { static async getUserByUsername(username) {
if (!username) return null if (!username) return null
const cachedUser = userCache.getByUsername(username) const normalizedUsername = username.toLowerCase()
const cachedUser = userCache.getByUsername(normalizedUsername)
if (cachedUser) return cachedUser if (cachedUser) return cachedUser
const user = await this.findOne({ const user = await this.findOne({
where: sequelize.where(sequelize.fn('lower', sequelize.col('username')), username.toLowerCase()), where: sequelize.where(sequelize.fn('LOWER', sequelize.col('username')), normalizedUsername),
include: this.sequelize.models.mediaProgress include: this.sequelize.models.mediaProgress
}) })
@ -369,11 +375,13 @@ class User extends Model {
static async getUserByEmail(email) { static async getUserByEmail(email) {
if (!email) return null if (!email) return null
const cachedUser = userCache.getByEmail(email) const normalizedEmail = email.toLowerCase()
const cachedUser = userCache.getByEmail(normalizedEmail)
if (cachedUser) return cachedUser if (cachedUser) return cachedUser
const user = await this.findOne({ const user = await this.findOne({
where: sequelize.where(sequelize.fn('lower', sequelize.col('email')), email.toLowerCase()), where: sequelize.where(sequelize.fn('LOWER', sequelize.col('email')), normalizedEmail),
include: this.sequelize.models.mediaProgress include: this.sequelize.models.mediaProgress
}) })
@ -414,10 +422,24 @@ class User extends Model {
const cachedUser = userCache.getById(userId) || userCache.getByOldId(userId) const cachedUser = userCache.getById(userId) || userCache.getByOldId(userId)
if (cachedUser) return cachedUser if (cachedUser) return cachedUser
const isUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userId)
if (isUuid) {
const byId = await this.findByPk(userId, {
include: this.sequelize.models.mediaProgress
})
if (byId) {
userCache.set(byId)
return byId
}
}
const oldIdMatcher =
this.sequelize.getDialect() === 'postgres'
? sequelize.where(sequelize.literal(`extradata#>>'{oldUserId}'`), userId)
: { 'extraData.oldUserId': userId }
const user = await this.findOne({ const user = await this.findOne({
where: { where: oldIdMatcher,
[sequelize.Op.or]: [{ id: userId }, { 'extraData.oldUserId': userId }]
},
include: this.sequelize.models.mediaProgress include: this.sequelize.models.mediaProgress
}) })

View file

@ -62,9 +62,9 @@ class Backup {
} }
} }
setData(backupDirPath) { setData(backupDirPath, dialect = 'sqlite') {
this.id = date.format(new Date(), 'YYYY-MM-DD[T]HHmm') this.id = date.format(new Date(), 'YYYY-MM-DD[T]HHmm')
this.key = 'sqlite' this.key = dialect
this.datePretty = date.format(new Date(), 'ddd, MMM D YYYY HH:mm') this.datePretty = date.format(new Date(), 'ddd, MMM D YYYY HH:mm')
this.backupDirPath = backupDirPath this.backupDirPath = backupDirPath

View file

@ -527,54 +527,128 @@ class ApiRouter {
return userSessions.sort((a, b) => b.updatedAt - a.updatedAt) return userSessions.sort((a, b) => b.updatedAt - a.updatedAt)
} }
async getUserListeningSessionsPageHelper(userId, page = 0, itemsPerPage = 10, mediaItemId = null) {
const where = { userId }
if (mediaItemId) where.mediaItemId = mediaItemId
const start = page * itemsPerPage
const [total, sessions] = await Promise.all([
Database.countPlaybackSessions(where),
Database.getPlaybackSessions(where, {
limit: itemsPerPage,
offset: start,
order: [['updatedAt', 'DESC']]
})
])
return {
total,
numPages: Math.ceil(total / itemsPerPage),
page,
itemsPerPage,
sessions
}
}
async getUserItemListeningSessionsHelper(userId, mediaItemId) { async getUserItemListeningSessionsHelper(userId, mediaItemId) {
const userSessions = await Database.getPlaybackSessions({ userId, mediaItemId }) const userSessions = await Database.getPlaybackSessions({ userId, mediaItemId })
return userSessions.sort((a, b) => b.updatedAt - a.updatedAt) return userSessions.sort((a, b) => b.updatedAt - a.updatedAt)
} }
async getUserListeningStatsHelpers(userId) { async getUserListeningStatsHelpers(userId, options = {}) {
const startedAt = Date.now()
const today = date.format(new Date(), 'YYYY-MM-DD') const today = date.format(new Date(), 'YYYY-MM-DD')
const includeItems = options.includeItems !== false
const includeRecentSessions = options.includeRecentSessions !== false
const getSessionField = (session, key) => {
if (!session || typeof session !== 'object') return undefined
if (session[key] !== undefined) return session[key]
const lowerKey = key.toLowerCase()
if (session[lowerKey] !== undefined) return session[lowerKey]
return undefined
}
const getSessionObjectField = (session, key) => {
const value = getSessionField(session, key)
if (!value) return null
if (typeof value === 'string') {
try {
return JSON.parse(value)
} catch (error) {
return null
}
}
return value
}
const tasks = [Database.getPlaybackSessionsForStats({ userId })]
if (includeRecentSessions) {
tasks.push(
Database.getPlaybackSessions({ userId }, {
limit: 10,
offset: 0,
order: [['updatedAt', 'DESC']]
})
)
}
const [listeningSessions, recentSessions = []] = await Promise.all(tasks)
const listeningSessions = await this.getUserListeningSessionsHelper(userId)
const listeningStats = { const listeningStats = {
totalTime: 0, totalTime: 0,
items: {},
days: {}, days: {},
dayOfWeek: {}, dayOfWeek: {},
today: 0, today: 0
recentSessions: listeningSessions.slice(0, 10)
} }
if (includeItems) listeningStats.items = {}
if (includeRecentSessions) listeningStats.recentSessions = recentSessions
listeningSessions.forEach((s) => { listeningSessions.forEach((s) => {
let sessionTimeListening = s.timeListening const extraData = getSessionObjectField(s, 'extraData')
if (typeof sessionTimeListening == 'string') { const libraryItemId = extraData?.libraryItemId || extraData?.libraryitemid || null
sessionTimeListening = Number(sessionTimeListening) const sessionDate = getSessionField(s, 'date')
} const sessionDayOfWeek = getSessionField(s, 'dayOfWeek')
const sessionUpdatedAt = getSessionField(s, 'updatedAt')
const sessionMediaMetadata = getSessionObjectField(s, 'mediaMetadata')
const numericListening = Number(getSessionField(s, 'timeListening'))
const sessionTimeListening = Number.isFinite(numericListening)
? numericListening
: 0
if (s.dayOfWeek) { if (sessionDayOfWeek) {
if (!listeningStats.dayOfWeek[s.dayOfWeek]) listeningStats.dayOfWeek[s.dayOfWeek] = 0 if (!listeningStats.dayOfWeek[sessionDayOfWeek]) listeningStats.dayOfWeek[sessionDayOfWeek] = 0
listeningStats.dayOfWeek[s.dayOfWeek] += sessionTimeListening listeningStats.dayOfWeek[sessionDayOfWeek] += sessionTimeListening
} }
if (s.date && sessionTimeListening > 0) { if (sessionDate && sessionTimeListening > 0) {
if (!listeningStats.days[s.date]) listeningStats.days[s.date] = 0 if (!listeningStats.days[sessionDate]) listeningStats.days[sessionDate] = 0
listeningStats.days[s.date] += sessionTimeListening listeningStats.days[sessionDate] += sessionTimeListening
if (s.date === today) { if (sessionDate === today) {
listeningStats.today += sessionTimeListening listeningStats.today += sessionTimeListening
} }
} }
if (!listeningStats.items[s.libraryItemId]) { if (!libraryItemId) {
listeningStats.items[s.libraryItemId] = { listeningStats.totalTime += sessionTimeListening
id: s.libraryItemId, return
timeListening: sessionTimeListening, }
mediaMetadata: s.mediaMetadata,
lastUpdate: s.lastUpdate if (includeItems) {
if (!listeningStats.items[libraryItemId]) {
listeningStats.items[libraryItemId] = {
id: libraryItemId,
timeListening: sessionTimeListening,
mediaMetadata: sessionMediaMetadata,
lastUpdate: sessionUpdatedAt
}
} else {
listeningStats.items[libraryItemId].timeListening += sessionTimeListening
} }
} else {
listeningStats.items[s.libraryItemId].timeListening += sessionTimeListening
} }
listeningStats.totalTime += sessionTimeListening listeningStats.totalTime += sessionTimeListening
}) })
Logger.debug(
`[ApiRouter] Listening stats for user "${userId}" aggregated ${listeningSessions.length} sessions in ${Date.now() - startedAt}ms includeItems=${includeItems} includeRecentSessions=${includeRecentSessions}`
)
return listeningStats return listeningStats
} }
} }

View file

@ -14,6 +14,7 @@ const LibraryItemScanner = require('./LibraryItemScanner')
const LibraryScan = require('./LibraryScan') const LibraryScan = require('./LibraryScan')
const LibraryItemScanData = require('./LibraryItemScanData') const LibraryItemScanData = require('./LibraryItemScanData')
const Task = require('../objects/Task') const Task = require('../objects/Task')
const { isPostgres, jsonArrayExpand } = require('../utils/sqlDialectHelpers')
class LibraryScanner { class LibraryScanner {
constructor() { constructor() {
@ -676,12 +677,16 @@ async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingle
// check if it was moved from another folder by comparing the ino to the library files // check if it was moved from another folder by comparing the ino to the library files
const ino = await fileUtils.getIno(fullPath) const ino = await fileUtils.getIno(fullPath)
if (!ino) return null if (!ino) return null
const inodeMatchQuery = isPostgres(Database.sequelize)
? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{ino}' = :inode)`
: `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)`
const existingLibraryItem = await Database.libraryItemModel.findOneExpanded( const existingLibraryItem = await Database.libraryItemModel.findOneExpanded(
[ [
{ {
libraryId: libraryId libraryId: libraryId
}, },
sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), { sequelize.where(sequelize.literal(inodeMatchQuery), {
[sequelize.Op.gt]: 0 [sequelize.Op.gt]: 0
}) })
], ],

View file

@ -0,0 +1,515 @@
#!/usr/bin/env node
const sqlite3 = require('sqlite3')
const { Client } = require('pg')
const SQLITE_PATH = process.env.SQLITE_PATH || '/config/absdatabase.sqlite'
const DATABASE_URL = process.env.DATABASE_URL
const PG_SCHEMA = process.env.PG_SCHEMA || 'public'
const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500)
const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true'
const ALLOW_DESTRUCTIVE_TARGET = String(process.env.ALLOW_DESTRUCTIVE_TARGET || 'false').toLowerCase() === 'true'
const integerBounds = {
smallint: { min: -32768n, max: 32767n },
integer: { min: -2147483648n, max: 2147483647n },
bigint: { min: -9223372036854775808n, max: 9223372036854775807n }
}
const preferredOrder = [
'migrationsMeta',
'SequelizeMeta',
'settings',
'users',
'apiKeys',
'sessions',
'libraries',
'libraryFolders',
'authors',
'series',
'books',
'podcasts',
'podcastEpisodes',
'libraryItems',
'bookAuthors',
'bookSeries',
'collections',
'collectionBooks',
'playlists',
'playlistMediaItems',
'mediaProgresses',
'devices',
'playbackSessions',
'feeds',
'feedEpisodes',
'mediaItemShares',
'customMetadataProviders'
]
function quoteIdent(identifier) {
return `"${String(identifier).replace(/"/g, '""')}"`
}
function openSqlite(filePath) {
return new Promise((resolve, reject) => {
const db = new sqlite3.Database(filePath, sqlite3.OPEN_READONLY, (err) => {
if (err) return reject(err)
resolve(db)
})
})
}
function sqliteAll(db, sql, params = []) {
return new Promise((resolve, reject) => {
db.all(sql, params, (err, rows) => {
if (err) return reject(err)
resolve(rows)
})
})
}
function sqliteGet(db, sql, params = []) {
return new Promise((resolve, reject) => {
db.get(sql, params, (err, row) => {
if (err) return reject(err)
resolve(row)
})
})
}
async function findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) {
const issues = []
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
const pgColumns = pgColumnsByTable.get(postgresTable)
if (!pgColumns) continue
for (const column of pgColumns.values()) {
if (column.data_type !== 'character varying' || !column.character_maximum_length) continue
const sqliteColumn = column.column_name
const maxLength = Number(column.character_maximum_length)
const quotedTable = quoteIdent(sqliteTable)
const quotedColumn = quoteIdent(sqliteColumn)
try {
const maxLengthRow = await sqliteGet(
sqliteDb,
`SELECT MAX(LENGTH(${quotedColumn})) AS maxLength FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`
)
const actualMaxLength = Number(maxLengthRow?.maxLength || 0)
if (actualMaxLength <= maxLength) continue
const overCountRow = await sqliteGet(
sqliteDb,
`SELECT COUNT(*) AS count FROM ${quotedTable} WHERE LENGTH(${quotedColumn}) > ?`,
[maxLength]
)
issues.push({
sqliteTable,
sqliteColumn,
postgresTable,
postgresColumn: column.column_name,
maxLength,
actualMaxLength,
overCount: Number(overCountRow?.count || 0)
})
} catch (error) {
// Ignore columns missing in sqlite source table
}
}
}
return issues
}
function parseIntegerValue(value) {
if (typeof value === 'number') {
if (!Number.isSafeInteger(value)) return null
return BigInt(value)
}
if (typeof value === 'string') {
const trimmed = value.trim()
if (!trimmed || !/^-?\d+$/.test(trimmed)) return null
try {
return BigInt(trimmed)
} catch (error) {
return null
}
}
return null
}
function isIntegerCompatible(value, dataType = 'bigint') {
if (value === null || value === undefined) return true
const parsed = parseIntegerValue(value)
if (parsed === null) return false
const bounds = integerBounds[dataType]
if (!bounds) return false
return parsed >= bounds.min && parsed <= bounds.max
}
async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) {
const issues = []
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
const pgColumns = pgColumnsByTable.get(postgresTable)
if (!pgColumns) continue
for (const column of pgColumns.values()) {
const dataType = column.data_type
if (dataType !== 'smallint' && dataType !== 'integer' && dataType !== 'bigint') continue
const sqliteColumn = column.column_name
const quotedTable = quoteIdent(sqliteTable)
const quotedColumn = quoteIdent(sqliteColumn)
let rows
try {
rows = await sqliteAll(sqliteDb, `SELECT ${quotedColumn} AS value FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`)
} catch (error) {
// Ignore columns missing in sqlite source table
continue
}
let badCount = 0
let sampleValue = null
for (const row of rows) {
if (!isIntegerCompatible(row.value, dataType)) {
badCount += 1
if (sampleValue === null) sampleValue = row.value
}
}
if (badCount > 0) {
issues.push({
sqliteTable,
sqliteColumn,
postgresTable,
postgresColumn: column.column_name,
postgresType: dataType,
badCount,
sampleValue
})
}
}
}
return issues
}
function normalizeBoolean(value) {
if (value === null || value === undefined) return null
if (typeof value === 'boolean') return value
if (typeof value === 'number') return value !== 0
if (typeof value === 'string') {
const v = value.trim().toLowerCase()
return v === 'true' || v === '1' || v === 't'
}
return !!value
}
function normalizeJson(value) {
if (value === null || value === undefined || value === '') return null
if (typeof value === 'object') return JSON.stringify(value)
const textValue = String(value)
try {
const parsed = JSON.parse(textValue)
if (typeof parsed === 'string') {
try {
return JSON.stringify(JSON.parse(parsed))
} catch (error) {
return JSON.stringify(parsed)
}
}
return JSON.stringify(parsed)
} catch (error) {
// Keep non-JSON payloads as JSON string values so inserts remain valid JSON.
return JSON.stringify(textValue)
}
}
function convertValue(value, pgColumn) {
if (!pgColumn) return value
const dataType = pgColumn.data_type
const udtName = pgColumn.udt_name
if (dataType === 'boolean') {
return normalizeBoolean(value)
}
if (dataType === 'json' || dataType === 'jsonb' || udtName === 'json' || udtName === 'jsonb') {
return normalizeJson(value)
}
if ((dataType === 'smallint' || dataType === 'integer' || dataType === 'bigint') && value !== null && value !== undefined) {
if (!isIntegerCompatible(value, dataType)) return value
if (dataType === 'bigint' && typeof value === 'string') {
return value.trim()
}
if (typeof value === 'number') return value
return Number(value)
}
return value
}
function getOverlongColumns(row, insertColumns) {
const overlong = []
for (const column of insertColumns) {
const maxLength = column.metadata.character_maximum_length
if (!maxLength) continue
const value = row[column.sqliteColumn]
if (value === null || value === undefined) continue
const length = String(value).length
if (length > maxLength) {
overlong.push({
sqliteColumn: column.sqliteColumn,
postgresColumn: column.postgresColumn,
maxLength,
actualLength: length
})
}
}
return overlong
}
async function main() {
if (!DATABASE_URL) {
throw new Error('DATABASE_URL is required')
}
console.log(`[migrate] sqlite source: ${SQLITE_PATH}`)
console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`)
console.log(`[migrate] dry run: ${DRY_RUN}`)
const sqliteDb = await openSqlite(SQLITE_PATH)
const pg = new Client({ connectionString: DATABASE_URL })
await pg.connect()
try {
const sqliteTablesRows = await sqliteAll(
sqliteDb,
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
)
const sqliteTables = sqliteTablesRows.map((row) => row.name)
const pgTablesRows = await pg.query(
`SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`,
[PG_SCHEMA]
)
const pgTablesByLowerName = new Map(pgTablesRows.rows.map((row) => [row.table_name.toLowerCase(), row.table_name]))
const tablesToMigrate = sqliteTables
.map((sqliteTable) => {
const postgresTable = pgTablesByLowerName.get(sqliteTable.toLowerCase())
if (!postgresTable) return null
return {
sqliteTable,
postgresTable
}
})
.filter(Boolean)
tablesToMigrate.sort((a, b) => {
const ai = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === a.sqliteTable.toLowerCase())
const bi = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === b.sqliteTable.toLowerCase())
if (ai === -1 && bi === -1) return a.sqliteTable.localeCompare(b.sqliteTable)
if (ai === -1) return 1
if (bi === -1) return -1
return ai - bi
})
if (!tablesToMigrate.length) {
throw new Error('No overlapping tables found between SQLite and PostgreSQL')
}
console.log(`[migrate] tables to migrate: ${tablesToMigrate.map((table) => table.sqliteTable).join(', ')}`)
const pgColumnsByTable = new Map()
for (const { postgresTable } of tablesToMigrate) {
const columnsResult = await pg.query(
`SELECT column_name, data_type, udt_name, character_maximum_length FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`,
[PG_SCHEMA, postgresTable]
)
const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column]))
pgColumnsByTable.set(postgresTable, columnMap)
}
const overlongVarcharIssues = await findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable)
const integerTypeIssues = await findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable)
if (overlongVarcharIssues.length) {
console.error('[migrate] overlong source values detected for varchar columns:')
overlongVarcharIssues.forEach((issue) => {
console.error(
`[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` +
`(max=${issue.maxLength}, actualMax=${issue.actualMaxLength}, overRows=${issue.overCount})`
)
})
throw new Error('Migration aborted to prevent truncation/data loss. Widen target column types first.')
}
if (integerTypeIssues.length) {
console.error('[migrate] non-integer source values detected for integer columns:')
integerTypeIssues.forEach((issue) => {
console.error(
`[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` +
`(type=${issue.postgresType}, badRows=${issue.badCount}, sample=${JSON.stringify(issue.sampleValue)})`
)
})
throw new Error('Migration aborted to prevent numeric precision loss. Widen target numeric column types first.')
}
if (!DRY_RUN) {
if (!ALLOW_DESTRUCTIVE_TARGET) {
throw new Error('Migration writes are destructive. Set ALLOW_DESTRUCTIVE_TARGET=true after confirming the target database can be truncated.')
}
await pg.query('BEGIN')
await pg.query('SET session_replication_role = replica')
const truncateList = tablesToMigrate.map(({ postgresTable }) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`).join(', ')
await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`)
console.log('[migrate] truncated target tables')
}
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(sqliteTable)}`)
const pgColumns = pgColumnsByTable.get(postgresTable)
const insertColumns = rows.length
? Object.keys(rows[0])
.map((sqliteColumn) => {
const pgColumn = pgColumns.get(sqliteColumn.toLowerCase())
if (!pgColumn) return null
return {
sqliteColumn,
postgresColumn: pgColumn.column_name,
metadata: pgColumn
}
})
.filter(Boolean)
: []
if (!rows.length || !insertColumns.length) {
console.log(`[migrate] ${sqliteTable}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`)
continue
}
if (!DRY_RUN) {
for (let offset = 0; offset < rows.length; offset += BATCH_SIZE) {
const batchRows = rows.slice(offset, offset + BATCH_SIZE)
const valuesSql = []
const params = []
let paramIndex = 1
for (const row of batchRows) {
const placeholders = []
for (const column of insertColumns) {
params.push(convertValue(row[column.sqliteColumn], column.metadata))
placeholders.push(`$${paramIndex++}`)
}
valuesSql.push(`(${placeholders.join(', ')})`)
}
const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}`
try {
await pg.query('SAVEPOINT migrate_batch')
await pg.query(insertSql, params)
await pg.query('RELEASE SAVEPOINT migrate_batch')
} catch (error) {
await pg.query('ROLLBACK TO SAVEPOINT migrate_batch')
console.error(`[migrate] batch insert failed for ${sqliteTable} (offset=${offset}, size=${batchRows.length}): ${error.message}`)
for (let rowIndex = 0; rowIndex < batchRows.length; rowIndex++) {
const row = batchRows[rowIndex]
const singleRowParams = insertColumns.map((column) => convertValue(row[column.sqliteColumn], column.metadata))
const singleRowInsertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES (${singleRowParams.map((_, index) => `$${index + 1}`).join(', ')})`
try {
await pg.query('SAVEPOINT migrate_row')
await pg.query(singleRowInsertSql, singleRowParams)
await pg.query('RELEASE SAVEPOINT migrate_row')
} catch (rowError) {
await pg.query('ROLLBACK TO SAVEPOINT migrate_row')
const overlongColumns = getOverlongColumns(row, insertColumns)
if (overlongColumns.length) {
overlongColumns.forEach((column) => {
console.error(
`[migrate] overlong value in ${sqliteTable}.${column.sqliteColumn} -> ${postgresTable}.${column.postgresColumn} ` +
`(length=${column.actualLength}, max=${column.maxLength})`
)
})
}
throw rowError
}
}
}
}
}
console.log(`[migrate] ${sqliteTable}: ${rows.length} rows`)
}
if (!DRY_RUN) {
await pg.query('SET session_replication_role = DEFAULT')
await pg.query('COMMIT')
console.log('[migrate] migration transaction committed')
}
const parity = []
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(sqliteTable)}`)
const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`)
parity.push({
table: sqliteTable,
sqliteCount: Number(sqliteCountRow.count || 0),
postgresCount: Number(pgCountResult.rows[0].count || 0)
})
}
const mismatches = parity.filter((row) => row.sqliteCount !== row.postgresCount)
if (mismatches.length) {
console.error('[migrate] row-count mismatches detected:')
mismatches.forEach((row) => {
console.error(`[migrate] ${row.table}: sqlite=${row.sqliteCount} postgres=${row.postgresCount}`)
})
process.exitCode = 2
} else {
console.log('[migrate] parity check passed for all migrated tables')
}
} finally {
sqliteDb.close()
await pg.end()
}
}
if (require.main === module) {
main().catch((error) => {
console.error('[migrate] failed:', error)
process.exit(1)
})
}
module.exports = {
normalizeJson,
isIntegerCompatible,
convertValue,
findOverlongVarcharValues,
findIntegerTypeIssues,
quoteIdent
}

View file

@ -5,10 +5,22 @@ const libraryItemsBookFilters = require('./libraryItemsBookFilters')
const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters') const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters')
const { createNewSortInstance } = require('../../libs/fastSort') const { createNewSortInstance } = require('../../libs/fastSort')
const { profile } = require('../../utils/profiler') const { profile } = require('../../utils/profiler')
const { booleanLiteral, jsonArrayContainsAny } = require('../sqlDialectHelpers')
const naturalSort = createNewSortInstance({ const naturalSort = createNewSortInstance({
comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
}) })
async function withShelfFallback(scope, fallbackValue, action) {
try {
return await action()
} catch (error) {
// Only postgres shelves fall back to empty results - on sqlite keep upstream behavior of surfacing the error
if (!Database.isPostgresDialect()) throw error
Logger.error(`[LibraryFilters] Failed to load ${scope}`, error)
return fallbackValue
}
}
module.exports = { module.exports = {
decode(text) { decode(text) {
return Buffer.from(decodeURIComponent(text), 'base64').toString() return Buffer.from(decodeURIComponent(text), 'base64').toString()
@ -49,22 +61,24 @@ module.exports = {
* @returns {Promise<{ items:import('../../models/LibraryItem')[], count:number }>} * @returns {Promise<{ items:import('../../models/LibraryItem')[], count:number }>}
*/ */
async getMediaItemsInProgress(library, user, include, limit) { async getMediaItemsInProgress(library, user, include, limit) {
if (library.isBook) { return withShelfFallback(`in-progress shelf for library "${library.id}"`, { items: [], count: 0 }, async () => {
const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'in-progress', 'progress', true, false, include, limit, 0, true) if (library.isBook) {
return { const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'in-progress', 'progress', true, false, include, limit, 0, true)
items: libraryItems.map((li) => { return {
const oldLibraryItem = li.toOldJSONMinified() items: libraryItems.map((li) => {
if (li.rssFeed) { const oldLibraryItem = li.toOldJSONMinified()
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() if (li.rssFeed) {
} oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
if (li.mediaItemShare) { }
oldLibraryItem.mediaItemShare = li.mediaItemShare if (li.mediaItemShare) {
} oldLibraryItem.mediaItemShare = li.mediaItemShare
return oldLibraryItem }
}), return oldLibraryItem
count }),
count
}
} }
} else {
const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'in-progress', 'progress', true, limit, 0, true) const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'in-progress', 'progress', true, limit, 0, true)
return { return {
count, count,
@ -74,7 +88,7 @@ module.exports = {
return oldLibraryItem return oldLibraryItem
}) })
} }
} })
}, },
/** /**
@ -86,25 +100,27 @@ module.exports = {
* @returns {object} { libraryItems:LibraryItem[], count:number } * @returns {object} { libraryItems:LibraryItem[], count:number }
*/ */
async getLibraryItemsMostRecentlyAdded(library, user, include, limit) { async getLibraryItemsMostRecentlyAdded(library, user, include, limit) {
if (library.isBook) { return withShelfFallback(`recent shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => {
const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, false, include, limit, 0) if (library.isBook) {
return { const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, false, include, limit, 0)
libraryItems: libraryItems.map((li) => { return {
const oldLibraryItem = li.toOldJSONMinified() libraryItems: libraryItems.map((li) => {
if (li.rssFeed) { const oldLibraryItem = li.toOldJSONMinified()
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() if (li.rssFeed) {
} oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
if (li.size && !oldLibraryItem.media.size) { }
oldLibraryItem.media.size = li.size if (li.size && !oldLibraryItem.media.size) {
} oldLibraryItem.media.size = li.size
if (li.mediaItemShare) { }
oldLibraryItem.mediaItemShare = li.mediaItemShare if (li.mediaItemShare) {
} oldLibraryItem.mediaItemShare = li.mediaItemShare
return oldLibraryItem }
}), return oldLibraryItem
count }),
count
}
} }
} else {
const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, include, limit, 0) const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, include, limit, 0)
return { return {
libraryItems: libraryItems.map((li) => { libraryItems: libraryItems.map((li) => {
@ -122,7 +138,7 @@ module.exports = {
}), }),
count count
} }
} })
}, },
/** /**
@ -134,23 +150,25 @@ module.exports = {
* @returns {object} { libraryItems:LibraryItem[], count:number } * @returns {object} { libraryItems:LibraryItem[], count:number }
*/ */
async getLibraryItemsContinueSeries(library, user, include, limit) { async getLibraryItemsContinueSeries(library, user, include, limit) {
const { libraryItems, count } = await libraryItemsBookFilters.getContinueSeriesLibraryItems(library, user, include, limit, 0) return withShelfFallback(`continue-series shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => {
return { const { libraryItems, count } = await libraryItemsBookFilters.getContinueSeriesLibraryItems(library, user, include, limit, 0)
libraryItems: libraryItems.map((li) => { return {
const oldLibraryItem = li.toOldJSONMinified() libraryItems: libraryItems.map((li) => {
if (li.rssFeed) { const oldLibraryItem = li.toOldJSONMinified()
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() if (li.rssFeed) {
} oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
if (li.series) { }
oldLibraryItem.media.metadata.series = li.series if (li.series) {
} oldLibraryItem.media.metadata.series = li.series
if (li.mediaItemShare) { }
oldLibraryItem.mediaItemShare = li.mediaItemShare if (li.mediaItemShare) {
} oldLibraryItem.mediaItemShare = li.mediaItemShare
return oldLibraryItem }
}), return oldLibraryItem
count }),
} count
}
})
}, },
/** /**
@ -163,22 +181,24 @@ module.exports = {
* @returns {Promise<{ items:oldLibraryItem[], count:number }>} * @returns {Promise<{ items:oldLibraryItem[], count:number }>}
*/ */
async getMediaFinished(library, user, include, limit) { async getMediaFinished(library, user, include, limit) {
if (library.isBook) { return withShelfFallback(`finished shelf for library "${library.id}"`, { items: [], count: 0 }, async () => {
const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'finished', 'progress', true, false, include, limit, 0) if (library.isBook) {
return { const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'finished', 'progress', true, false, include, limit, 0)
items: libraryItems.map((li) => { return {
const oldLibraryItem = li.toOldJSONMinified() items: libraryItems.map((li) => {
if (li.rssFeed) { const oldLibraryItem = li.toOldJSONMinified()
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() if (li.rssFeed) {
} oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
if (li.mediaItemShare) { }
oldLibraryItem.mediaItemShare = li.mediaItemShare if (li.mediaItemShare) {
} oldLibraryItem.mediaItemShare = li.mediaItemShare
return oldLibraryItem }
}), return oldLibraryItem
count }),
count
}
} }
} else {
const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'finished', 'progress', true, limit, 0) const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'finished', 'progress', true, limit, 0)
return { return {
count, count,
@ -188,7 +208,7 @@ module.exports = {
return oldLibraryItem return oldLibraryItem
}) })
} }
} })
}, },
/** /**
@ -202,23 +222,24 @@ module.exports = {
async getSeriesMostRecentlyAdded(library, user, include, limit) { async getSeriesMostRecentlyAdded(library, user, include, limit) {
if (!library.isBook) return { series: [], count: 0 } if (!library.isBook) return { series: [], count: 0 }
const seriesIncludes = [] return withShelfFallback(`recent-series shelf for library "${library.id}"`, { series: [], count: 0 }, async () => {
if (include.includes('rssfeed')) { const seriesIncludes = []
seriesIncludes.push({ if (include.includes('rssfeed')) {
model: Database.feedModel seriesIncludes.push({
}) model: Database.feedModel
} })
const userPermissionBookWhere = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user)
const seriesWhere = [
{
libraryId: library.id,
createdAt: {
[Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
}
} }
]
const userPermissionBookWhere = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user)
const seriesWhere = [
{
libraryId: library.id,
createdAt: {
[Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
}
}
]
// Handle library setting to hide single book series // Handle library setting to hide single book series
// TODO: Merge with existing query // TODO: Merge with existing query
@ -235,13 +256,13 @@ module.exports = {
if (userPermissionBookWhere.bookWhere.length) { if (userPermissionBookWhere.bookWhere.length) {
let attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id' let attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id'
if (!user.canAccessExplicitContent) { if (!user.canAccessExplicitContent) {
attrQuery += ' AND b.explicit = 0' attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}`
} }
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
if (user.permissions.selectedTagsNotAccessible) { if (user.permissions.selectedTagsNotAccessible) {
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0' attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0`
} else { } else {
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0' attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0`
} }
} }
seriesWhere.push( seriesWhere.push(
@ -251,29 +272,29 @@ module.exports = {
) )
} }
const { rows: series, count } = await Database.seriesModel.findAndCountAll({ const { rows: series, count } = await Database.seriesModel.findAndCountAll({
where: seriesWhere, where: seriesWhere,
limit, limit,
offset: 0, offset: 0,
distinct: true, distinct: true,
subQuery: false, subQuery: false,
replacements: userPermissionBookWhere.replacements, replacements: userPermissionBookWhere.replacements,
include: [ include: [
{ {
model: Database.bookSeriesModel, model: Database.bookSeriesModel,
include: {
model: Database.bookModel,
where: userPermissionBookWhere.bookWhere,
include: { include: {
model: Database.libraryItemModel model: Database.bookModel,
} where: userPermissionBookWhere.bookWhere,
include: {
model: Database.libraryItemModel
}
},
separate: true
}, },
separate: true ...seriesIncludes
}, ],
...seriesIncludes order: [['createdAt', 'DESC']]
], })
order: [['createdAt', 'DESC']]
})
const allOldSeries = [] const allOldSeries = []
for (const s of series) { for (const s of series) {
@ -311,10 +332,11 @@ module.exports = {
allOldSeries.push(oldSeries) allOldSeries.push(oldSeries)
} }
return { return {
series: allOldSeries, series: allOldSeries,
count count
} }
})
}, },
/** /**
@ -329,37 +351,39 @@ module.exports = {
async getNewestAuthors(library, user, limit) { async getNewestAuthors(library, user, limit) {
if (library.mediaType !== 'book') return { authors: [], count: 0 } if (library.mediaType !== 'book') return { authors: [], count: 0 }
const { bookWhere, replacements } = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user) return withShelfFallback(`newest-authors shelf for library "${library.id}"`, { authors: [], count: 0 }, async () => {
const { bookWhere, replacements } = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user)
const { rows: authors, count } = await Database.authorModel.findAndCountAll({ const { rows: authors, count } = await Database.authorModel.findAndCountAll({
where: { where: {
libraryId: library.id, libraryId: library.id,
createdAt: { createdAt: {
[Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
} }
}, },
replacements, replacements,
include: { include: {
model: Database.bookModel, model: Database.bookModel,
attributes: ['id', 'tags', 'explicit'], attributes: ['id', 'tags', 'explicit'],
where: bookWhere, where: bookWhere,
required: true, // Must belong to a book required: true, // Must belong to a book
through: { through: {
attributes: [] attributes: []
} }
}, },
limit, limit,
distinct: true, distinct: true,
order: [['createdAt', 'DESC']] order: [['createdAt', 'DESC']]
})
return {
authors: authors.map((au) => {
const numBooks = au.books.length || 0
return au.toOldJSONExpanded(numBooks)
}),
count
}
}) })
return {
authors: authors.map((au) => {
const numBooks = au.books.length || 0
return au.toOldJSONExpanded(numBooks)
}),
count
}
}, },
/** /**
@ -373,20 +397,22 @@ module.exports = {
async getLibraryItemsToDiscover(library, user, include, limit) { async getLibraryItemsToDiscover(library, user, include, limit) {
if (library.mediaType !== 'book') return { libraryItems: [], count: 0 } if (library.mediaType !== 'book') return { libraryItems: [], count: 0 }
const { libraryItems, count } = await libraryItemsBookFilters.getDiscoverLibraryItems(library.id, user, include, limit) return withShelfFallback(`discover shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => {
return { const { libraryItems, count } = await libraryItemsBookFilters.getDiscoverLibraryItems(library.id, user, include, limit)
libraryItems: libraryItems.map((li) => { return {
const oldLibraryItem = li.toOldJSONMinified() libraryItems: libraryItems.map((li) => {
if (li.rssFeed) { const oldLibraryItem = li.toOldJSONMinified()
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified() if (li.rssFeed) {
} oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
if (li.mediaItemShare) { }
oldLibraryItem.mediaItemShare = li.mediaItemShare if (li.mediaItemShare) {
} oldLibraryItem.mediaItemShare = li.mediaItemShare
return oldLibraryItem }
}), return oldLibraryItem
count }),
} count
}
})
}, },
/** /**
@ -399,15 +425,17 @@ module.exports = {
async getNewestPodcastEpisodes(library, user, limit) { async getNewestPodcastEpisodes(library, user, limit) {
if (library.mediaType !== 'podcast') return { libraryItems: [], count: 0 } if (library.mediaType !== 'podcast') return { libraryItems: [], count: 0 }
const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'recent', null, 'createdAt', true, limit, 0) return withShelfFallback(`newest-podcast-episodes shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => {
return { const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'recent', null, 'createdAt', true, limit, 0)
count, return {
libraryItems: libraryItems.map((li) => { count,
const oldLibraryItem = li.toOldJSONMinified() libraryItems: libraryItems.map((li) => {
oldLibraryItem.recentEpisode = li.recentEpisode const oldLibraryItem = li.toOldJSONMinified()
return oldLibraryItem oldLibraryItem.recentEpisode = li.recentEpisode
}) return oldLibraryItem
} })
}
})
}, },
/** /**

View file

@ -2,6 +2,7 @@ const Sequelize = require('sequelize')
const Database = require('../../Database') const Database = require('../../Database')
const libraryItemsBookFilters = require('./libraryItemsBookFilters') const libraryItemsBookFilters = require('./libraryItemsBookFilters')
const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters') const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters')
const { jsonArrayContainsAny } = require('../sqlDialectHelpers')
module.exports = { module.exports = {
/** /**
@ -12,7 +13,7 @@ module.exports = {
async getAllLibraryItemsWithTags(tags) { async getAllLibraryItemsWithTags(tags) {
const libraryItems = [] const libraryItems = []
const booksWithTag = await Database.bookModel.findAll({ const booksWithTag = await Database.bookModel.findAll({
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), { where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}), }),
replacements: { replacements: {
@ -46,7 +47,7 @@ module.exports = {
libraryItems.push(libraryItem) libraryItems.push(libraryItem)
} }
const podcastsWithTag = await Database.podcastModel.findAll({ const podcastsWithTag = await Database.podcastModel.findAll({
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), { where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}), }),
replacements: { replacements: {
@ -77,7 +78,7 @@ module.exports = {
async getAllLibraryItemsWithGenres(genres) { async getAllLibraryItemsWithGenres(genres) {
const libraryItems = [] const libraryItems = []
const booksWithGenre = await Database.bookModel.findAll({ const booksWithGenre = await Database.bookModel.findAll({
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), { where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}), }),
replacements: { replacements: {
@ -107,7 +108,7 @@ module.exports = {
libraryItems.push(libraryItem) libraryItems.push(libraryItem)
} }
const podcastsWithGenre = await Database.podcastModel.findAll({ const podcastsWithGenre = await Database.podcastModel.findAll({
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), { where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}), }),
replacements: { replacements: {
@ -138,7 +139,7 @@ module.exports = {
async getAllLibraryItemsWithNarrators(narrators) { async getAllLibraryItemsWithNarrators(narrators) {
const libraryItems = [] const libraryItems = []
const booksWithGenre = await Database.bookModel.findAll({ const booksWithGenre = await Database.bookModel.findAll({
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(narrators) WHERE json_valid(narrators) AND json_each.value IN (:narrators))`), { where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('narrators', 'narrators', Database.sequelize)), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}), }),
replacements: { replacements: {

View file

@ -6,6 +6,7 @@ const authorFilters = require('./authorFilters')
const ShareManager = require('../../managers/ShareManager') const ShareManager = require('../../managers/ShareManager')
const { profile } = require('../profiler') const { profile } = require('../profiler')
const stringifySequelizeQuery = require('../stringifySequelizeQuery') const stringifySequelizeQuery = require('../stringifySequelizeQuery')
const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand, safeTextToDoubleExpression, safeTextToIntegerExpression } = require('../sqlDialectHelpers')
const countCache = new Map() const countCache = new Map()
module.exports = { module.exports = {
@ -27,10 +28,10 @@ module.exports = {
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
replacements['userTagsSelected'] = user.permissions.itemTagsSelected replacements['userTagsSelected'] = user.permissions.itemTagsSelected
if (user.permissions.selectedTagsNotAccessible) { if (user.permissions.selectedTagsNotAccessible) {
bookWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0)) bookWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0))
} else { } else {
bookWhere.push( bookWhere.push(
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), { Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}) })
) )
@ -189,7 +190,7 @@ module.exports = {
} else if (group === 'explicit') { } else if (group === 'explicit') {
mediaWhere['explicit'] = true mediaWhere['explicit'] = true
} else if (['genres', 'tags', 'narrators'].includes(group)) { } else if (['genres', 'tags', 'narrators'].includes(group)) {
mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), { mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}) })
replacements.filterValue = value replacements.filterValue = value
@ -236,7 +237,7 @@ module.exports = {
} else if (group === 'publishedDecades') { } else if (group === 'publishedDecades') {
const startYear = parseInt(value) const startYear = parseInt(value)
const endYear = parseInt(value, 10) + 9 const endYear = parseInt(value, 10) + 9
mediaWhere = Sequelize.where(Sequelize.literal('CAST(publishedYear AS INTEGER)'), { mediaWhere = Sequelize.where(Sequelize.literal(safeTextToIntegerExpression('publishedYear', Database.sequelize)), {
[Sequelize.Op.between]: [startYear, endYear] [Sequelize.Op.between]: [startYear, endYear]
}) })
} }
@ -256,9 +257,9 @@ module.exports = {
const getTitleOrder = () => { const getTitleOrder = () => {
if (global.ServerSettings.sortingIgnorePrefix) { if (global.ServerSettings.sortingIgnorePrefix) {
return [Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir] return [Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]
} else { } else {
return [Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir] return [Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]
} }
} }
@ -273,21 +274,22 @@ module.exports = {
} else if (sortBy === 'media.duration') { } else if (sortBy === 'media.duration') {
return [['duration', dir]] return [['duration', dir]]
} else if (sortBy === 'media.metadata.publishedYear') { } else if (sortBy === 'media.metadata.publishedYear') {
return [[Sequelize.literal(`CAST(\`book\`.\`publishedYear\` AS INTEGER)`), dir]] return [[Sequelize.literal(safeTextToIntegerExpression('book.publishedYear', Database.sequelize)), dir]]
} else if (sortBy === 'media.metadata.authorNameLF') { } else if (sortBy === 'media.metadata.authorNameLF') {
// Sort by author name last first, secondary sort by title // Sort by author name last first, secondary sort by title
return [[Sequelize.literal('`libraryItem`.`authorNamesLastFirst` COLLATE NOCASE'), dir], getTitleOrder()] return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesLastFirst', Database.sequelize)), dir], getTitleOrder()]
} else if (sortBy === 'media.metadata.authorName') { } else if (sortBy === 'media.metadata.authorName') {
// Sort by author name first last, secondary sort by title // Sort by author name first last, secondary sort by title
return [[Sequelize.literal('`libraryItem`.`authorNamesFirstLast` COLLATE NOCASE'), dir], getTitleOrder()] return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesFirstLast', Database.sequelize)), dir], getTitleOrder()]
} else if (sortBy === 'media.metadata.title') { } else if (sortBy === 'media.metadata.title') {
if (collapseseries) { if (collapseseries) {
return [[Sequelize.literal('display_title COLLATE NOCASE'), dir]] return [[Sequelize.literal(noCaseSortExpression('display_title', Database.sequelize)), dir]]
} }
return [getTitleOrder()] return [getTitleOrder()]
} else if (sortBy === 'sequence') { } else if (sortBy === 'sequence') {
const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'
return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]] const sequenceColumn = Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`'
return [[Sequelize.literal(`${safeTextToDoubleExpression(sequenceColumn, Database.sequelize)} ${nullDir}`)]]
} else if (sortBy === 'progress') { } else if (sortBy === 'progress') {
return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]] return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]]
} else if (sortBy === 'progress.createdAt') { } else if (sortBy === 'progress.createdAt') {
@ -326,7 +328,9 @@ module.exports = {
required: true required: true
} }
], ],
order: [Sequelize.literal('CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')] order: [
Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"books->bookSeries"."sequence"' : '`books.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`)
]
}) })
const bookSeriesToInclude = [] const bookSeriesToInclude = []
const booksToInclude = [] const booksToInclude = []
@ -358,26 +362,32 @@ module.exports = {
async findAndCountAll(findOptions, limit, offset, useCountCache) { async findAndCountAll(findOptions, limit, offset, useCountCache) {
const model = Database.bookModel const model = Database.bookModel
if (useCountCache) {
const countCacheKey = stringifySequelizeQuery(findOptions)
Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`)
if (!countCache.has(countCacheKey)) {
const count = await model.count(findOptions)
countCache.set(countCacheKey, count)
}
findOptions.limit = limit || null
findOptions.offset = offset
const rows = await model.findAll(findOptions)
return { rows, count: countCache.get(countCacheKey) }
}
findOptions.limit = limit || null findOptions.limit = limit || null
findOptions.offset = offset findOptions.offset = offset
return await model.findAndCountAll(findOptions) try {
if (useCountCache) {
const countCacheKey = stringifySequelizeQuery(findOptions)
Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`)
if (!countCache.has(countCacheKey)) {
const count = await model.count(findOptions)
countCache.set(countCacheKey, count)
}
const rows = await model.findAll(findOptions)
return { rows, count: countCache.get(countCacheKey) }
}
return await model.findAndCountAll(findOptions)
} catch (error) {
Logger.error(`[LibraryItemsBookFilters] findAndCountAll failed: ${error.message}`)
try {
Logger.error(`[LibraryItemsBookFilters] findAndCountAll query: ${stringifySequelizeQuery(findOptions)}`)
} catch (stringifyError) {
Logger.error(`[LibraryItemsBookFilters] failed to stringify query: ${stringifyError.message}`)
}
throw error
}
}, },
/** /**
@ -455,12 +465,28 @@ module.exports = {
}) })
} else if (filterGroup === 'ebooks' && filterValue === 'supplementary') { } else if (filterGroup === 'ebooks' && filterValue === 'supplementary') {
// TODO: Temp workaround for filtering supplementary ebook // TODO: Temp workaround for filtering supplementary ebook
libraryItemWhere['libraryFiles'] = { if (Database.sequelize.getDialect() === 'postgres') {
[Sequelize.Op.substring]: `"isSupplementary":true` libraryItemWhere[Sequelize.Op.and] = [
Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), {
[Sequelize.Op.like]: '%"isSupplementary":true%'
})
]
} else {
libraryItemWhere['libraryFiles'] = {
[Sequelize.Op.substring]: `"isSupplementary":true`
}
} }
} else if (filterGroup === 'ebooks' && filterValue === 'no-supplementary') { } else if (filterGroup === 'ebooks' && filterValue === 'no-supplementary') {
libraryItemWhere['libraryFiles'] = { if (Database.sequelize.getDialect() === 'postgres') {
[Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`) libraryItemWhere[Sequelize.Op.and] = [
Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), {
[Sequelize.Op.notLike]: '%"isSupplementary":true%'
})
]
} else {
libraryItemWhere['libraryFiles'] = {
[Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`)
}
} }
} else if (filterGroup === 'missing' && filterValue === 'authors') { } else if (filterGroup === 'missing' && filterValue === 'authors') {
authorInclude = { authorInclude = {
@ -502,7 +528,9 @@ module.exports = {
}) })
if (sortBy !== 'sequence') { if (sortBy !== 'sequence') {
// Secondary sort by sequence // Secondary sort by sequence
sortOrder.push([Sequelize.literal('CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]) sortOrder.push([
Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`)
])
} }
} else if (filterGroup === 'issues') { } else if (filterGroup === 'issues') {
libraryItemWhere[Sequelize.Op.or] = [ libraryItemWhere[Sequelize.Op.or] = [
@ -594,10 +622,32 @@ module.exports = {
// When collapsing series and sorting by title then use the series name instead of the book title // When collapsing series and sorting by title then use the series name instead of the book title
// for this set an attribute "display_title" to use in sorting // for this set an attribute "display_title" to use in sorting
const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.title' : '`libraryItem`.`title`'
const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.titleIgnorePrefix' : '`libraryItem`.`titleIgnorePrefix`'
const fallbackFn = coalesceFunctionName(Database.sequelize)
const includedBookSeriesIds = bookSeriesToInclude.map((v) => Database.sequelize.escape(v.id)).join(', ')
const collapseSeriesSubqueryByName = includedBookSeriesIds
? `(SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))`
: 'NULL'
const collapseSeriesSubqueryByNameIgnorePrefix = includedBookSeriesIds
? `(SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))`
: 'NULL'
Logger.debug(
`[LibraryItemsBookFilters] collapse-series computed includeIds=${bookSeriesToInclude.length} excludeBooks=${booksToExclude.length} ` +
`filterGroup=${filterGroup || 'none'} filterValue=${filterValue || 'none'} sortBy=${sortBy}`
)
if (!includedBookSeriesIds) {
Logger.debug(
`[LibraryItemsBookFilters] collapse-series produced no include IDs; using library item title fallback ` +
`(libraryId=${libraryId}, filterGroup=${filterGroup || 'none'}, filterValue=${filterValue || 'none'}, sortBy=${sortBy})`
)
}
if (global.ServerSettings.sortingIgnorePrefix) { if (global.ServerSettings.sortingIgnorePrefix) {
bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`titleIgnorePrefix\`)`), 'display_title']) bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByNameIgnorePrefix}, ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title'])
} else { } else {
bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`title\`)`), 'display_title']) bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByName}, ${fallbackLibraryItemTitle})`), 'display_title'])
} }
} }
@ -716,13 +766,14 @@ module.exports = {
bookWhere.push(...userPermissionBookWhere.bookWhere) bookWhere.push(...userPermissionBookWhere.bookWhere)
let includeAttributes = [[Sequelize.literal('(SELECT max(mp.updatedAt) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id)'), 'recent_progress']] let includeAttributes = [[Sequelize.literal('(SELECT max(mp.updatedAt) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id)'), 'recent_progress']]
let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = 0 OR mp.isFinished IS NULL)` let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = ${booleanLiteral(false, Database.sequelize)} OR mp.isFinished IS NULL)`
if (library.settings.onlyShowLaterBooksInContinueSeries) { if (library.settings.onlyShowLaterBooksInContinueSeries) {
const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as FLOAT) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = 1 AND mp.userId = :userId AND bs.seriesId = series.id)` const safeSequenceExpr = safeTextToDoubleExpression('bs.sequence', Database.sequelize)
const maxSequenceQuery = `(SELECT max(${safeSequenceExpr}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)`
includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence']) includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence'])
booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as FLOAT) > ${maxSequenceQuery}` booksNotFinishedQuery = booksNotFinishedQuery + ` AND ${safeSequenceExpr} > ${maxSequenceQuery}`
} }
const { rows: series, count } = await Database.seriesModel.findAndCountAll({ const { rows: series, count } = await Database.seriesModel.findAndCountAll({
@ -735,7 +786,7 @@ module.exports = {
}, },
// TODO: Simplify queries // TODO: Simplify queries
// Has at least 1 book finished // Has at least 1 book finished
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = 1)`), { Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)})`), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}), }),
// Has at least 1 book not finished (that has a sequence number higher than the highest already read, if library config is toggled) // Has at least 1 book not finished (that has a sequence number higher than the highest already read, if library config is toggled)
@ -743,7 +794,7 @@ module.exports = {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}), }),
// Has no books in progress // Has no books in progress
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = 0 AND mp.currentTime > 0)`), 0) Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)} AND mp.currentTime > 0)`), 0)
], ],
attributes: { attributes: {
include: includeAttributes include: includeAttributes
@ -757,10 +808,10 @@ module.exports = {
attributes: ['bookId', 'sequence'], attributes: ['bookId', 'sequence'],
separate: true, separate: true,
subQuery: false, subQuery: false,
order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]],
where: { where: {
'$book.mediaProgresses.isFinished$': { '$book.mediaProgresses.isFinished$': {
[Sequelize.Op.or]: [null, 0] [Sequelize.Op.or]: [null, false]
} }
}, },
include: { include: {
@ -854,7 +905,7 @@ module.exports = {
{ {
libraryId libraryId
}, },
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = 1 OR mp.currentTime > 0))`), 0) Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0))`), 0)
], ],
replacements: { replacements: {
userId: user.id, userId: user.id,
@ -870,7 +921,7 @@ module.exports = {
model: Database.bookModel, model: Database.bookModel,
where: userPermissionBookWhere.bookWhere where: userPermissionBookWhere.bookWhere
}, },
order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]], order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]],
limit: 1 limit: 1
}, },
subQuery: false, subQuery: false,
@ -891,7 +942,7 @@ module.exports = {
const discoverWhere = [ const discoverWhere = [
{ {
'$mediaProgresses.isFinished$': { '$mediaProgresses.isFinished$': {
[Sequelize.Op.or]: [null, 0] [Sequelize.Op.or]: [null, false]
}, },
'$mediaProgresses.currentTime$': { '$mediaProgresses.currentTime$': {
[Sequelize.Op.or]: [null, 0] [Sequelize.Op.or]: [null, 0]
@ -1174,7 +1225,7 @@ module.exports = {
// Search narrators // Search narrators
const narratorMatches = [] const narratorMatches = []
const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, { const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, ${jsonArrayExpand('b.narrators', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
replacements: { replacements: {
libraryId: library.id, libraryId: library.id,
limit, limit,
@ -1191,7 +1242,7 @@ module.exports = {
// Search tags // Search tags
const tagMatches = [] const tagMatches = []
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.tags', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: { replacements: {
libraryId: library.id, libraryId: library.id,
limit, limit,
@ -1208,7 +1259,7 @@ module.exports = {
// Search genres // Search genres
const genreMatches = [] const genreMatches = []
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: { replacements: {
libraryId: library.id, libraryId: library.id,
limit, limit,
@ -1288,7 +1339,7 @@ module.exports = {
*/ */
async getGenresWithCount(libraryId) { async getGenresWithCount(libraryId) {
const genres = [] const genres = []
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
replacements: { replacements: {
libraryId libraryId
}, },

View file

@ -3,6 +3,7 @@ const Database = require('../../Database')
const Logger = require('../../Logger') const Logger = require('../../Logger')
const { profile } = require('../../utils/profiler') const { profile } = require('../../utils/profiler')
const stringifySequelizeQuery = require('../stringifySequelizeQuery') const stringifySequelizeQuery = require('../stringifySequelizeQuery')
const { jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression, jsonArrayExpand, jsonPathNumber } = require('../sqlDialectHelpers')
const countCache = new Map() const countCache = new Map()
@ -24,10 +25,10 @@ module.exports = {
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
replacements['userTagsSelected'] = user.permissions.itemTagsSelected replacements['userTagsSelected'] = user.permissions.itemTagsSelected
if (user.permissions.selectedTagsNotAccessible) { if (user.permissions.selectedTagsNotAccessible) {
podcastWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0)) podcastWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0))
} else { } else {
podcastWhere.push( podcastWhere.push(
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), { Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}) })
) )
@ -53,7 +54,7 @@ module.exports = {
const replacements = {} const replacements = {}
if (['genres', 'tags'].includes(group)) { if (['genres', 'tags'].includes(group)) {
mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), { mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), {
[Sequelize.Op.gte]: 1 [Sequelize.Op.gte]: 1
}) })
replacements.filterValue = value replacements.filterValue = value
@ -87,12 +88,12 @@ module.exports = {
return [[Sequelize.literal('libraryItem.mtime'), dir]] return [[Sequelize.literal('libraryItem.mtime'), dir]]
} else if (sortBy === 'media.metadata.author') { } else if (sortBy === 'media.metadata.author') {
const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST' const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'
return [[Sequelize.literal(`\`podcast\`.\`author\` COLLATE NOCASE ${nullDir}`)]] return [[Sequelize.literal(`${noCaseSortExpression('podcast.author', Database.sequelize)} ${nullDir}`)]]
} else if (sortBy === 'media.metadata.title') { } else if (sortBy === 'media.metadata.title') {
if (global.ServerSettings.sortingIgnorePrefix) { if (global.ServerSettings.sortingIgnorePrefix) {
return [[Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]] return [[Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]]
} else { } else {
return [[Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]] return [[Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]]
} }
} else if (sortBy === 'media.numTracks') { } else if (sortBy === 'media.numTracks') {
return [['numEpisodes', dir]] return [['numEpisodes', dir]]
@ -455,7 +456,7 @@ module.exports = {
// Search tags // Search tags
const tagMatches = [] const tagMatches = []
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.tags', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: { replacements: {
libraryId: library.id, libraryId: library.id,
limit, limit,
@ -472,7 +473,7 @@ module.exports = {
// Search genres // Search genres
const genreMatches = [] const genreMatches = []
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, { const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: { replacements: {
libraryId: library.id, libraryId: library.id,
limit, limit,
@ -563,12 +564,12 @@ module.exports = {
* @returns {Promise<{ totalSize:number, totalDuration:number, numAudioFiles:number, totalItems:number}>} * @returns {Promise<{ totalSize:number, totalDuration:number, numAudioFiles:number, totalItems:number}>}
*/ */
async getPodcastLibraryStats(libraryId) { async getPodcastLibraryStats(libraryId) {
const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = "podcast" AND li.libraryId = :libraryId;`, { const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = 'podcast' AND li.libraryId = :libraryId;`, {
replacements: { replacements: {
libraryId libraryId
} }
}) })
const [statResults] = await Database.sequelize.query(`SELECT SUM(json_extract(pe.audioFile, '$.duration')) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, { const [statResults] = await Database.sequelize.query(`SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, {
replacements: { replacements: {
libraryId libraryId
} }
@ -588,7 +589,7 @@ module.exports = {
*/ */
async getGenresWithCount(libraryId) { async getGenresWithCount(libraryId) {
const genres = [] const genres = []
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, { const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
replacements: { replacements: {
libraryId libraryId
}, },
@ -611,7 +612,7 @@ module.exports = {
*/ */
async getLongestPodcasts(libraryId, limit) { async getLongestPodcasts(libraryId, limit) {
const podcasts = await Database.podcastModel.findAll({ const podcasts = await Database.podcastModel.findAll({
attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(json_extract(pe.audioFile, '$.duration')) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']], attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']],
include: { include: {
model: Database.libraryItemModel, model: Database.libraryItemModel,
attributes: ['id', 'libraryId'], attributes: ['id', 'libraryId'],

View file

@ -2,6 +2,7 @@ const Sequelize = require('sequelize')
const Logger = require('../../Logger') const Logger = require('../../Logger')
const Database = require('../../Database') const Database = require('../../Database')
const libraryItemsBookFilters = require('./libraryItemsBookFilters') const libraryItemsBookFilters = require('./libraryItemsBookFilters')
const { booleanLiteral, jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression } = require('../sqlDialectHelpers')
module.exports = { module.exports = {
decode(text) { decode(text) {
@ -60,7 +61,7 @@ module.exports = {
// TODO: Simplify and break-out // TODO: Simplify and break-out
let attrQuery = null let attrQuery = null
if (['genres', 'tags', 'narrators'].includes(filterGroup)) { if (['genres', 'tags', 'narrators'].includes(filterGroup)) {
attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (SELECT count(*) FROM json_each(b.${filterGroup}) WHERE json_valid(b.${filterGroup}) AND json_each.value = :filterValue) > 0` attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ${jsonArrayContainsValue(`b.${filterGroup}`, 'filterValue', Database.sequelize)} > 0`
userPermissionBookWhere.replacements.filterValue = filterValue userPermissionBookWhere.replacements.filterValue = filterValue
} else if (filterGroup === 'authors') { } else if (filterGroup === 'authors') {
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs, bookAuthors ba WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ba.bookId = b.id AND ba.authorId = :filterValue' attrQuery = 'SELECT count(*) FROM books b, bookSeries bs, bookAuthors ba WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ba.bookId = b.id AND ba.authorId = :filterValue'
@ -73,18 +74,18 @@ module.exports = {
userPermissionBookWhere.replacements.filterValue = filterValue userPermissionBookWhere.replacements.filterValue = filterValue
} else if (filterGroup === 'progress') { } else if (filterGroup === 'progress') {
if (filterValue === 'not-finished') { if (filterValue === 'not-finished') {
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)' attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})`
userPermissionBookWhere.replacements.userId = user.id userPermissionBookWhere.replacements.userId = user.id
} else if (filterValue === 'finished') { } else if (filterValue === 'finished') {
const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)' const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})`
seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0)) seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0))
userPermissionBookWhere.replacements.userId = user.id userPermissionBookWhere.replacements.userId = user.id
} else if (filterValue === 'not-started') { } else if (filterValue === 'not-started') {
const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = 1 OR mp.currentTime > 0)' const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0)`
seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0)) seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0))
userPermissionBookWhere.replacements.userId = user.id userPermissionBookWhere.replacements.userId = user.id
} else if (filterValue === 'in-progress') { } else if (filterValue === 'in-progress') {
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = 0' attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)}`
userPermissionBookWhere.replacements.userId = user.id userPermissionBookWhere.replacements.userId = user.id
} }
} }
@ -95,13 +96,13 @@ module.exports = {
if (!attrQuery) attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id' if (!attrQuery) attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id'
if (!user.canAccessExplicitContent) { if (!user.canAccessExplicitContent) {
attrQuery += ' AND b.explicit = 0' attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}`
} }
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) { if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
if (user.permissions.selectedTagsNotAccessible) { if (user.permissions.selectedTagsNotAccessible) {
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0' attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0`
} else { } else {
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0' attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0`
} }
} }
} }
@ -128,9 +129,9 @@ module.exports = {
order.push(['createdAt', dir]) order.push(['createdAt', dir])
} else if (sortBy === 'name') { } else if (sortBy === 'name') {
if (global.ServerSettings.sortingIgnorePrefix) { if (global.ServerSettings.sortingIgnorePrefix) {
order.push([Sequelize.literal('nameIgnorePrefix COLLATE NOCASE'), dir]) order.push([Sequelize.literal(noCaseSortExpression('nameIgnorePrefix', Database.sequelize)), dir])
} else { } else {
order.push([Sequelize.literal('`series`.`name` COLLATE NOCASE'), dir]) order.push([Sequelize.literal(noCaseSortExpression('series.name', Database.sequelize)), dir])
} }
} else if (sortBy === 'totalDuration') { } else if (sortBy === 'totalDuration') {
seriesAttributes.include.push([Sequelize.literal('(SELECT SUM(b.duration) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND b.id = bs.bookId)'), 'totalDuration']) seriesAttributes.include.push([Sequelize.literal('(SELECT SUM(b.duration) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND b.id = bs.bookId)'), 'totalDuration'])

View file

@ -0,0 +1,91 @@
function getDialect(sequelize) {
if (!sequelize) return 'sqlite'
if (typeof sequelize.getDialect === 'function') return sequelize.getDialect()
return sequelize.dialect?.name || 'sqlite'
}
function isPostgres(sequelize) {
return getDialect(sequelize) === 'postgres'
}
function booleanLiteral(value, sequelize) {
if (isPostgres(sequelize)) return value ? 'TRUE' : 'FALSE'
return value ? '1' : '0'
}
function noCaseSortExpression(columnExpression, sequelize) {
if (isPostgres(sequelize)) return `LOWER(${columnExpression})`
return `${columnExpression} COLLATE NOCASE`
}
function coalesceFunctionName(sequelize) {
return isPostgres(sequelize) ? 'COALESCE' : 'IFNULL'
}
function jsonArrayContainsAny(columnExpression, bindName, sequelize) {
if (isPostgres(sequelize)) {
return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:${bindName}))`
}
return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value IN (:${bindName}))`
}
function jsonArrayContainsValue(columnExpression, bindName, sequelize) {
if (isPostgres(sequelize)) {
return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :${bindName})`
}
return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value = :${bindName})`
}
function jsonArrayExpand(columnExpression, sequelize, options = {}) {
const alias = options.alias || 'json_each'
const textValues = options.textValues !== false
if (isPostgres(sequelize)) {
const fn = textValues ? 'jsonb_array_elements_text' : 'jsonb_array_elements'
return `${fn}(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS ${alias}(value)`
}
return `json_each(${columnExpression})`
}
function jsonPathText(columnExpression, pathSegments, sequelize) {
const path = Array.isArray(pathSegments) ? pathSegments : [pathSegments]
if (isPostgres(sequelize)) {
return `${columnExpression}::jsonb #>> '{${path.join(',')}}'`
}
return `json_extract(${columnExpression}, '$.${path.join('.')}')`
}
function jsonPathNumber(columnExpression, pathSegments, sequelize) {
if (isPostgres(sequelize)) {
return `NULLIF(${jsonPathText(columnExpression, pathSegments, sequelize)}, '')::double precision`
}
return `json_extract(${columnExpression}, '$.${[].concat(pathSegments).join('.')}')`
}
function safeTextToDoubleExpression(columnExpression, sequelize) {
if (isPostgres(sequelize)) {
return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(${columnExpression})::double precision ELSE NULL END`
}
return `CAST(${columnExpression} AS FLOAT)`
}
function safeTextToIntegerExpression(columnExpression, sequelize) {
if (isPostgres(sequelize)) {
return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?\\d+$' THEN BTRIM(${columnExpression})::integer ELSE NULL END`
}
return `CAST(${columnExpression} AS INTEGER)`
}
module.exports = {
getDialect,
isPostgres,
booleanLiteral,
noCaseSortExpression,
coalesceFunctionName,
jsonArrayContainsAny,
jsonArrayContainsValue,
jsonArrayExpand,
jsonPathText,
jsonPathNumber,
safeTextToDoubleExpression,
safeTextToIntegerExpression
}

View file

@ -0,0 +1,160 @@
const { expect } = require('chai')
const sinon = require('sinon')
const fs = require('../../server/libs/fsExtra')
const Database = require('../../server/Database')
describe('Database', () => {
let originalDialect
let originalSequelize
let originalEnv
beforeEach(() => {
originalDialect = Database.dialect
originalSequelize = Database.sequelize
originalEnv = {
DB_DIALECT: process.env.DB_DIALECT,
DATABASE_URL: process.env.DATABASE_URL
}
})
afterEach(() => {
Database.dialect = originalDialect
Database.sequelize = originalSequelize
if (originalEnv.DB_DIALECT === undefined) delete process.env.DB_DIALECT
else process.env.DB_DIALECT = originalEnv.DB_DIALECT
if (originalEnv.DATABASE_URL === undefined) delete process.env.DATABASE_URL
else process.env.DATABASE_URL = originalEnv.DATABASE_URL
sinon.restore()
})
describe('getConfiguredDialect', () => {
it('should default to sqlite when no env variables are set', () => {
delete process.env.DB_DIALECT
delete process.env.DATABASE_URL
expect(Database.getConfiguredDialect()).to.equal('sqlite')
})
it('should use explicit DB_DIALECT value when valid', () => {
process.env.DB_DIALECT = 'postgres'
process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite'
expect(Database.getConfiguredDialect()).to.equal('postgres')
})
it('should infer postgres dialect from DATABASE_URL', () => {
delete process.env.DB_DIALECT
process.env.DATABASE_URL = 'postgresql://user:pass@localhost:5432/abs'
expect(Database.getConfiguredDialect()).to.equal('postgres')
})
it('should fallback to sqlite for unsupported DB_DIALECT', () => {
process.env.DB_DIALECT = 'mysql'
process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite'
expect(Database.getConfiguredDialect()).to.equal('sqlite')
})
})
describe('checkHasDb', () => {
it('should not check sqlite file existence in postgres mode', async () => {
Database.dialect = 'postgres'
const pathExistsStub = sinon.stub(fs, 'pathExists')
const hasDb = await Database.checkHasDb()
expect(hasDb).to.equal(true)
expect(pathExistsStub.called).to.equal(false)
})
})
describe('checkHasTables', () => {
it('should return true when at least one table exists', async () => {
Database.sequelize = {
getQueryInterface: () => ({
showAllTables: sinon.stub().resolves(['users'])
})
}
const hasTables = await Database.checkHasTables()
expect(hasTables).to.equal(true)
})
it('should return false when no tables exist', async () => {
Database.sequelize = {
getQueryInterface: () => ({
showAllTables: sinon.stub().resolves([])
})
}
const hasTables = await Database.checkHasTables()
expect(hasTables).to.equal(false)
})
})
describe('addPostgresTriggers', () => {
function captureQueries(existingTriggers = []) {
const queries = []
Database.sequelize = {
query: async (sql) => {
queries.push(sql)
const count = existingTriggers.filter((name) => sql.includes(`tgname = '${name}'`)).length
return [[{ count }]]
}
}
return queries
}
it('should create title and author names triggers with folded lowercase identifiers', async () => {
const queries = captureQueries()
await Database.addPostgresTriggers()
const functions = queries.filter((sql) => sql.includes('CREATE OR REPLACE FUNCTION'))
const triggers = queries.filter((sql) => sql.includes('CREATE TRIGGER'))
expect(functions.length).to.equal(7)
expect(triggers.length).to.equal(7)
const allDdl = [...functions, ...triggers].join('\n')
// No camelCase identifiers may leak into postgres DDL - unquoted identifiers fold to lowercase
expect(allDdl).to.not.match(/libraryItems|bookAuthors|mediaId|titleIgnorePrefix|authorNames|bookId|authorId|lastFirst|createdAt/)
const authorTrigger = triggers.find((sql) => sql.includes('update_library_items_author_names_on_authors_update'))
expect(authorTrigger).to.include('AFTER UPDATE OF name ON authors')
const insertFn = functions.find((sql) => sql.includes('update_library_items_author_names_on_book_authors_insert_fn'))
expect(insertFn).to.include("string_agg(authors.name, ', ' ORDER BY bookauthors.createdat ASC)")
expect(insertFn).to.include('WHERE mediaid = NEW.bookid')
const deleteFn = functions.find((sql) => sql.includes('update_library_items_author_names_on_book_authors_delete_fn'))
expect(deleteFn).to.include('WHERE mediaid = OLD.bookid')
})
it('should skip triggers that already exist', async () => {
const queries = captureQueries(['update_library_items_title'])
await Database.addPostgresTriggers()
const titleFn = queries.find((sql) => sql.includes('update_library_items_title_fn'))
expect(titleFn).to.equal(undefined)
const otherFns = queries.filter((sql) => sql.includes('CREATE OR REPLACE FUNCTION'))
expect(otherFns.length).to.equal(6)
})
it('should dispatch to postgres triggers from addTriggers', async () => {
Database.dialect = 'postgres'
const queries = captureQueries()
await Database.addTriggers()
expect(queries.some((sql) => sql.includes('CREATE TRIGGER'))).to.equal(true)
})
})
})

View file

@ -0,0 +1,189 @@
const { expect } = require('chai')
const sinon = require('sinon')
const os = require('os')
const Path = require('path')
const EventEmitter = require('events')
const childProcess = require('child_process')
const sqlite3 = require('sqlite3')
const BackupManager = require('../../../server/managers/BackupManager')
const Backup = require('../../../server/objects/Backup')
const Database = require('../../../server/Database')
describe('BackupManager', () => {
let originalDialect
let originalDbPath
let originalConfigPath
let originalMetadataPath
beforeEach(() => {
originalDialect = Database.dialect
originalDbPath = Database.dbPath
originalConfigPath = global.ConfigPath
originalMetadataPath = global.MetadataPath
global.MetadataPath = os.tmpdir()
})
afterEach(() => {
Database.dialect = originalDialect
Database.dbPath = originalDbPath
global.ConfigPath = originalConfigPath
global.MetadataPath = originalMetadataPath
sinon.restore()
})
it('should select Postgres custom-format backups for the Postgres dialect', () => {
Database.dialect = 'postgres'
const manager = new BackupManager()
expect(manager.databaseBackupConfig).to.deep.equal({
dialect: 'postgres',
entryName: 'absdatabase.postgres.dump'
})
})
it('should create Postgres dumps with pg_dump without exposing credentials in argv', async () => {
Database.dialect = 'postgres'
Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf'
global.ConfigPath = os.tmpdir()
const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => {
callback(null, '', '')
})
const manager = new BackupManager()
const backup = new Backup()
backup.id = '2026-08-02T0130'
const dumpPath = await manager.backupPostgresDb(backup)
expect(dumpPath).to.equal(Path.join(os.tmpdir(), 'absdatabase.2026-08-02T0130.postgres.dump'))
expect(execFileStub.calledOnce).to.equal(true)
expect(execFileStub.firstCall.args[0]).to.equal('pg_dump')
expect(execFileStub.firstCall.args[1]).to.deep.equal([
'--format=custom',
'--no-owner',
'--no-acl',
'--file',
dumpPath,
'--host',
'localhost',
'--dbname',
'audiobookshelf',
'--port',
'5432',
'--username',
'absuser'
])
expect(execFileStub.firstCall.args[1].join(' ')).to.not.include('secretpass')
expect(execFileStub.firstCall.args[2].env.PGPASSWORD).to.equal('secretpass')
expect(execFileStub.firstCall.args[2].timeout).to.be.a('number')
})
it('should restore Postgres dumps in one transaction and clean existing objects', async () => {
Database.dialect = 'postgres'
Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf'
const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => {
callback(null, '', '')
})
const manager = new BackupManager()
await manager.restorePostgresDb('/config/absdatabase-postgres-temp.dump')
expect(execFileStub.firstCall.args[0]).to.equal('pg_restore')
expect(execFileStub.firstCall.args[1]).to.deep.equal([
'--clean',
'--if-exists',
'--exit-on-error',
'--single-transaction',
'--no-owner',
'--no-acl',
'/config/absdatabase-postgres-temp.dump',
'--host',
'localhost',
'--dbname',
'audiobookshelf',
'--port',
'5432',
'--username',
'absuser'
])
expect(execFileStub.firstCall.args[1].join(' ')).to.not.include('secretpass')
expect(execFileStub.firstCall.args[2].env.PGPASSWORD).to.equal('secretpass')
})
it('should redact database credentials from failed pg command errors', async () => {
Database.dialect = 'postgres'
Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf'
global.ConfigPath = os.tmpdir()
sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => {
const error = new Error('Command failed: pg_dump --dbname postgresql://absuser:secretpass@localhost/audiobookshelf\npg_dump: error: password authentication failed')
error.cmd = 'pg_dump --dbname postgresql://absuser:secretpass@localhost/audiobookshelf'
callback(error, '', 'connection using password secretpass failed')
})
const manager = new BackupManager()
const backup = new Backup()
backup.id = '2026-08-02T0130'
let error
try {
await manager.backupPostgresDb(backup)
} catch (caughtError) {
error = caughtError
}
expect(error).to.be.an('error')
expect(error.message).to.not.include('secretpass')
expect(error.cmd).to.not.include('secretpass')
expect(error.stderr).to.not.include('secretpass')
expect(error.message).to.include('***')
})
it('should reject pg commands when DATABASE_URL is not a valid URI', async () => {
Database.dialect = 'postgres'
Database.dbPath = 'not a connection uri'
global.ConfigPath = os.tmpdir()
const manager = new BackupManager()
const backup = new Backup()
backup.id = '2026-08-02T0130'
let error
try {
await manager.backupPostgresDb(backup)
} catch (caughtError) {
error = caughtError
}
expect(error).to.be.an('error')
expect(error.message).to.include('valid postgres connection URI')
})
it('should reject SQLite backup open errors without an uncaught sqlite event', async () => {
Database.dialect = 'sqlite'
Database.dbPath = '/config/absdatabase.sqlite'
global.ConfigPath = os.tmpdir()
sinon.stub(sqlite3, 'Database').callsFake(function (_dbPath, callback) {
const db = new EventEmitter()
db.close = (closeCallback) => closeCallback()
process.nextTick(() => callback(Object.assign(new Error('unable to open database file'), { code: 'SQLITE_CANTOPEN' })))
return db
})
const manager = new BackupManager()
const backup = new Backup()
backup.id = '2026-08-02T0130'
let error
try {
await manager.backupSqliteDb(backup)
} catch (caughtError) {
error = caughtError
}
expect(error).to.be.an('error')
expect(error.code).to.equal('SQLITE_CANTOPEN')
})
})

View file

@ -189,6 +189,52 @@ describe('MigrationManager', () => {
expect(loggerInfoStub.calledWith(sinon.match('Restored the original database'))).to.be.true expect(loggerInfoStub.calledWith(sinon.match('Restored the original database'))).to.be.true
expect(processExitStub.calledOnce).to.be.true expect(processExitStub.calledOnce).to.be.true
}) })
it('should skip sqlite backup workflow for postgres migrations', async () => {
// Arrange
migrationManager.serverVersion = '1.2.0'
migrationManager.databaseVersion = '1.1.0'
migrationManager.maxVersion = '1.1.0'
migrationManager.initialized = true
sequelizeStub.getDialect.returns('postgres')
umzugStub.migrations.resolves([{ name: 'v1.2.0-migration.js' }])
umzugStub.executed.resolves([{ name: 'v1.1.0-migration.js' }])
// Act
await migrationManager.runMigrations()
// Assert
expect(umzugStub.up.calledOnce).to.be.true
expect(fsCopyStub.called).to.be.false
expect(fsRemoveStub.called).to.be.false
})
})
describe('tableExists', () => {
it('should use queryInterface.tableExists when available', async () => {
const tableExistsStub = sinon.stub().resolves(true)
const sequelize = sinon.createStubInstance(Sequelize)
sequelize.getQueryInterface.returns({ tableExists: tableExistsStub })
const manager = new MigrationManager(sequelize, false, configPath)
const exists = await manager.tableExists('migrationsMeta')
expect(exists).to.equal(true)
expect(tableExistsStub.calledOnceWithExactly('migrationsMeta')).to.equal(true)
})
it('should fallback to showAllTables and support object table names', async () => {
const sequelize = sinon.createStubInstance(Sequelize)
sequelize.getQueryInterface.returns({
showAllTables: sinon.stub().resolves([{ tableName: 'migrationsMeta' }])
})
const manager = new MigrationManager(sequelize, false, configPath)
const exists = await manager.tableExists('migrationsMeta')
expect(exists).to.equal(true)
})
}) })
describe('fetchVersionsFromDatabase', () => { describe('fetchVersionsFromDatabase', () => {
@ -280,6 +326,20 @@ describe('MigrationManager', () => {
expect(error.message).to.equal('Database query failed') expect(error.message).to.equal('Database query failed')
} }
}) })
it('should support lowercase maxversion alias from postgres drivers', async () => {
const sequelize = sinon.createStubInstance(Sequelize)
sequelize.query.onFirstCall().resolves([{ version: '1.1.0' }])
sequelize.query.onSecondCall().resolves([{ maxversion: '1.2.0' }])
const manager = new MigrationManager(sequelize, false, configPath)
manager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves()
await manager.fetchVersionsFromDatabase()
expect(manager.databaseVersion).to.equal('1.1.0')
expect(manager.maxVersion).to.equal('1.2.0')
})
}) })
describe('updateMaxVersion', () => { describe('updateMaxVersion', () => {

View file

@ -88,6 +88,23 @@ describe('migration-v2.15.0-series-column-unique', () => {
await queryInterface.dropTable('Series') await queryInterface.dropTable('Series')
await queryInterface.dropTable('BookSeries') await queryInterface.dropTable('BookSeries')
}) })
it('should skip NOCASE reindex on postgres dialect', async () => {
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
const originalQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize)
const queryStub = sinon.stub(queryInterface.sequelize, 'query').callsFake((sql, options) => {
if (sql === 'REINDEX NOCASE;') {
throw new Error('Unexpected NOCASE reindex in postgres mode')
}
return originalQuery(sql, options)
})
await up({ context: { queryInterface, logger: Logger } })
expect(queryStub.neverCalledWith('REINDEX NOCASE;')).to.be.true
expect(loggerInfoStub.calledWith(sinon.match('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.be.true
})
it('upgrade with no duplicate series', async () => { it('upgrade with no duplicate series', async () => {
// Add some entries to the Series table using the UUID for the ids // Add some entries to the Series table using the UUID for the ids
await queryInterface.bulkInsert('Series', [ await queryInterface.bulkInsert('Series', [

View file

@ -0,0 +1,41 @@
const { expect } = require('chai')
const sinon = require('sinon')
const { up, down } = require('../../../server/migrations/v2.15.1-reindex-nocase')
const Logger = require('../../../server/Logger')
describe('migration-v2.15.1-reindex-nocase', () => {
afterEach(() => {
sinon.restore()
})
it('should skip reindex on non-sqlite dialect', async () => {
const queryStub = sinon.stub().resolves()
const loggerInfoStub = sinon.stub(Logger, 'info')
const queryInterface = {
sequelize: {
getDialect: () => 'postgres',
query: queryStub
}
}
await up({ context: { queryInterface, logger: Logger } })
expect(queryStub.called).to.equal(false)
expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.equal(true)
})
it('should log no-op on down migration', async () => {
const loggerInfoStub = sinon.stub(Logger, 'info')
const queryInterface = {
sequelize: {
getDialect: () => 'postgres',
query: sinon.stub().resolves()
}
}
await down({ context: { queryInterface, logger: Logger } })
expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] No action required for downgrade'))).to.equal(true)
})
})

View file

@ -0,0 +1,98 @@
const { expect } = require('chai')
const { Sequelize, DataTypes } = require('sequelize')
const sinon = require('sinon')
const Logger = require('../../../server/Logger')
const { up, down } = require('../../../server/migrations/v2.15.2-index-creation')
describe('migration-v2.15.2-index-creation', () => {
let sequelize
let queryInterface
beforeEach(async () => {
sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
queryInterface = sequelize.getQueryInterface()
await queryInterface.createTable('bookAuthors', {
id: { type: DataTypes.INTEGER, primaryKey: true },
authorId: { type: DataTypes.INTEGER }
})
await queryInterface.createTable('bookSeries', {
id: { type: DataTypes.INTEGER, primaryKey: true },
seriesId: { type: DataTypes.INTEGER }
})
await queryInterface.createTable('podcastEpisodes', {
id: { type: DataTypes.INTEGER, primaryKey: true },
createdAt: { type: DataTypes.DATE },
podcastId: { type: DataTypes.INTEGER }
})
})
afterEach(async () => {
if (sequelize) await sequelize.close()
})
it('up should succeed when legacy podcast index is missing', async () => {
await up({ context: { queryInterface, logger: Logger } })
const indexes = await queryInterface.showIndex('podcastEpisodes')
expect(indexes.some((index) => index.name === 'podcastEpisode_createdAt_podcastId')).to.equal(true)
})
it('down should succeed when new podcast index is missing', async () => {
await down({ context: { queryInterface, logger: Logger } })
const indexes = await queryInterface.showIndex('podcastEpisodes')
expect(indexes.some((index) => index.name === 'podcast_episodes_created_at')).to.equal(true)
})
it('up should treat index names case-insensitively', async () => {
const qi = {
showIndex: sinon.stub(),
addIndex: sinon.stub().resolves(),
removeIndex: sinon.stub().resolves()
}
qi.showIndex.onCall(0).resolves([{ name: 'bookauthor_authorid' }])
qi.showIndex.onCall(1).resolves([{ name: 'bookseries_seriesid' }])
qi.showIndex.onCall(2).resolves([{ name: 'podcast_episodes_created_at' }])
qi.showIndex.onCall(3).resolves([{ name: 'podcastepisode_createdat_podcastid' }])
await up({ context: { queryInterface: qi, logger: Logger } })
expect(qi.addIndex.called).to.equal(false)
expect(qi.removeIndex.calledOnceWithExactly('podcastEpisodes', 'podcast_episodes_created_at')).to.equal(true)
})
it('up should continue when addIndex reports already exists', async () => {
const makePgExistsError = (sql) => {
const err = new Error('relation already exists')
err.name = 'SequelizeDatabaseError'
err.original = { code: '42P07' }
err.sql = sql
return err
}
const qi = {
showIndex: sinon.stub(),
addIndex: sinon.stub(),
removeIndex: sinon.stub().resolves()
}
qi.showIndex.onCall(0).resolves([])
qi.showIndex.onCall(1).resolves([])
qi.showIndex.onCall(2).resolves([{ name: 'podcast_episodes_created_at' }])
qi.showIndex.onCall(3).resolves([])
qi.addIndex.onCall(0).rejects(makePgExistsError('CREATE INDEX bookAuthor_authorId ON bookAuthors (authorId)'))
qi.addIndex.onCall(1).rejects(makePgExistsError('CREATE INDEX bookSeries_seriesId ON bookSeries (seriesId)'))
qi.addIndex.onCall(2).rejects(makePgExistsError('CREATE INDEX podcastEpisode_createdAt_podcastId ON podcastEpisodes (createdAt, podcastId)'))
await up({ context: { queryInterface: qi, logger: Logger } })
expect(qi.addIndex.callCount).to.equal(3)
expect(qi.removeIndex.calledOnceWithExactly('podcastEpisodes', 'podcast_episodes_created_at')).to.equal(true)
})
})

View file

@ -34,6 +34,16 @@ describe('migration-v2.17.3-fk-constraints', () => {
await queryInterface.dropAllTables() await queryInterface.dropAllTables()
}) })
it('should skip sqlite-specific migration on non-sqlite dialect', async () => {
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
const queryStub = sinon.stub(queryInterface.sequelize, 'query').resolves([])
await up({ context: { queryInterface, logger: Logger } })
expect(queryStub.called).to.equal(false)
expect(loggerInfoStub.calledWith(sinon.match('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect'))).to.equal(true)
})
it('should fix table foreign key constraints', async () => { it('should fix table foreign key constraints', async () => {
// Create tables with missing foreign key constraints: libraryItems, feeds, mediaItemShares, playbackSessions, playlistMediaItems, mediaProgresses // Create tables with missing foreign key constraints: libraryItems, feeds, mediaItemShares, playbackSessions, playlistMediaItems, mediaProgresses
await queryInterface.sequelize.query('CREATE TABLE `libraryItems` (`id` UUID UNIQUE PRIMARY KEY, `libraryId` UUID REFERENCES `libraries` (`id`), `libraryFolderId` UUID REFERENCES `libraryFolders` (`id`));') await queryInterface.sequelize.query('CREATE TABLE `libraryItems` (`id` UUID UNIQUE PRIMARY KEY, `libraryId` UUID REFERENCES `libraries` (`id`), `libraryFolderId` UUID REFERENCES `libraryFolders` (`id`));')

View file

@ -30,11 +30,16 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
expect(logger.info.calledWith('[2.17.4 migration] UPGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] UPGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] OIDC is enabled, adding authOpenIDSubfolderForRedirectURLs to server settings')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] OIDC is enabled, adding authOpenIDSubfolderForRedirectURLs to server settings')).to.be.true
expect(queryInterface.sequelize.query.calledTwice).to.be.true expect(queryInterface.sequelize.query.calledTwice).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true
expect( expect(
queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = "server-settings";', { queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(
queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', {
replacements: { replacements: {
value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }) value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }),
settingsKey: 'server-settings'
} }
}) })
).to.be.true ).to.be.true
@ -49,10 +54,31 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
expect(logger.info.calledWith('[2.17.4 migration] UPGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] UPGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] OIDC is not enabled, no action required')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] OIDC is not enabled, no action required')).to.be.true
expect(queryInterface.sequelize.query.calledOnce).to.be.true expect(queryInterface.sequelize.query.calledOnce).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true expect(
queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] UPGRADE END: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] UPGRADE END: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true
}) })
it('should handle already-parsed object server settings', async () => {
queryInterface.sequelize.query.onFirstCall().resolves([[{ value: { authActiveAuthMethods: ['openid'] } }]])
queryInterface.sequelize.query.onSecondCall().resolves()
await up({ context })
expect(queryInterface.sequelize.query.calledTwice).to.be.true
expect(
queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', {
replacements: {
value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }),
settingsKey: 'server-settings'
}
})
).to.be.true
})
it('should throw an error if server settings cannot be parsed', async () => { it('should throw an error if server settings cannot be parsed', async () => {
queryInterface.sequelize.query.onFirstCall().resolves([[{ value: 'invalid json' }]]) queryInterface.sequelize.query.onFirstCall().resolves([[{ value: 'invalid json' }]])
@ -60,7 +86,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
await up({ context }) await up({ context })
} catch (error) { } catch (error) {
expect(queryInterface.sequelize.query.calledOnce).to.be.true expect(queryInterface.sequelize.query.calledOnce).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true expect(
queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(logger.error.calledWith('[2.17.4 migration] Error parsing server settings:')).to.be.true expect(logger.error.calledWith('[2.17.4 migration] Error parsing server settings:')).to.be.true
expect(error).to.be.instanceOf(Error) expect(error).to.be.instanceOf(Error)
} }
@ -73,7 +103,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
await up({ context }) await up({ context })
} catch (error) { } catch (error) {
expect(queryInterface.sequelize.query.calledOnce).to.be.true expect(queryInterface.sequelize.query.calledOnce).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true expect(
queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(logger.error.calledWith('[2.17.4 migration] Server settings not found')).to.be.true expect(logger.error.calledWith('[2.17.4 migration] Server settings not found')).to.be.true
expect(error).to.be.instanceOf(Error) expect(error).to.be.instanceOf(Error)
} }
@ -90,11 +124,16 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] Removing authOpenIDSubfolderForRedirectURLs from server settings')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] Removing authOpenIDSubfolderForRedirectURLs from server settings')).to.be.true
expect(queryInterface.sequelize.query.calledTwice).to.be.true expect(queryInterface.sequelize.query.calledTwice).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true
expect( expect(
queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = "server-settings";', { queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(
queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', {
replacements: { replacements: {
value: JSON.stringify({}) value: JSON.stringify({}),
settingsKey: 'server-settings'
} }
}) })
).to.be.true ).to.be.true
@ -109,7 +148,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] authOpenIDSubfolderForRedirectURLs not found in server settings, no action required')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] authOpenIDSubfolderForRedirectURLs not found in server settings, no action required')).to.be.true
expect(queryInterface.sequelize.query.calledOnce).to.be.true expect(queryInterface.sequelize.query.calledOnce).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true expect(
queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE END: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE END: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true
}) })
}) })

View file

@ -47,6 +47,18 @@ describe('Migration v2.19.1-copy-title-to-library-items', () => {
}) })
describe('up', () => { describe('up', () => {
it('should skip sqlite-specific migration on non-sqlite dialect', async () => {
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
await up({ context: { queryInterface, logger: Logger } })
const table = await queryInterface.describeTable('libraryItems')
expect(table).to.have.property('id')
expect(table).to.not.have.property('title')
expect(table).to.not.have.property('titleIgnorePrefix')
expect(loggerInfoStub.calledWith(sinon.match('[2.19.1 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true
})
it('should copy title and titleIgnorePrefix to libraryItems', async () => { it('should copy title and titleIgnorePrefix to libraryItems', async () => {
await up({ context: { queryInterface, logger: Logger } }) await up({ context: { queryInterface, logger: Logger } })

View file

@ -75,6 +75,18 @@ describe('Migration v2.19.4-improve-podcast-queries', () => {
}) })
describe('up', () => { describe('up', () => {
it('should skip sqlite-specific migration on non-sqlite dialect', async () => {
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
await up({ context: { queryInterface, logger: Logger } })
const podcastsTable = await queryInterface.describeTable('podcasts')
const mediaProgressesTable = await queryInterface.describeTable('mediaProgresses')
expect(podcastsTable).to.not.have.property('numEpisodes')
expect(mediaProgressesTable).to.not.have.property('podcastId')
expect(loggerInfoStub.calledWith(sinon.match('[2.19.4 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true
})
it('should add numEpisodes column to podcasts', async () => { it('should add numEpisodes column to podcasts', async () => {
await up({ context: { queryInterface, logger: Logger } }) await up({ context: { queryInterface, logger: Logger } })

View file

@ -0,0 +1,98 @@
const { expect } = require('chai')
const sinon = require('sinon')
const { up, down } = require('../../../server/migrations/v2.35.0-add-last-refresh-token')
const { Sequelize } = require('sequelize')
const Logger = require('../../../server/Logger')
describe('migration-v2.35.0-add-last-refresh-token', () => {
let sequelize
let queryInterface
let loggerInfoStub
beforeEach(() => {
sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
queryInterface = sequelize.getQueryInterface()
loggerInfoStub = sinon.stub(Logger, 'info')
})
afterEach(() => {
sinon.restore()
})
describe('up', () => {
beforeEach(async () => {
await queryInterface.createTable('sessions', {
id: { type: Sequelize.UUID, primaryKey: true },
createdAt: { type: Sequelize.DATE, allowNull: false },
updatedAt: { type: Sequelize.DATE, allowNull: false }
})
})
it('should add lastRefreshToken columns when they do not exist', async () => {
await up({ context: { queryInterface, logger: Logger } })
const tableDescription = await queryInterface.describeTable('sessions')
expect(tableDescription.lastRefreshToken).to.exist
expect(tableDescription.lastRefreshTokenExpiresAt).to.exist
})
it('should not fail when the columns already exist', async () => {
await up({ context: { queryInterface, logger: Logger } })
await up({ context: { queryInterface, logger: Logger } })
const tableDescription = await queryInterface.describeTable('sessions')
expect(tableDescription.lastRefreshToken).to.exist
expect(tableDescription.lastRefreshTokenExpiresAt).to.exist
expect(loggerInfoStub.calledWithMatch('lastRefreshToken column already exists')).to.equal(true)
})
it('should detect existing columns case-insensitively for postgres identifier folding', async () => {
// Postgres folds unquoted identifiers to lowercase, so describeTable on a
// migrated postgres database returns lowercase column names
const fakeQueryInterface = {
sequelize,
tableExists: async () => true,
describeTable: async () => ({
id: {},
lastrefreshtoken: {},
lastrefreshtokenexpiresat: {}
}),
addColumn: async () => {
throw new Error('addColumn must not be called for existing lowercase columns')
}
}
await up({ context: { queryInterface: fakeQueryInterface, logger: Logger } })
expect(loggerInfoStub.calledWithMatch('lastRefreshToken column already exists')).to.equal(true)
expect(loggerInfoStub.calledWithMatch('lastRefreshTokenExpiresAt column already exists')).to.equal(true)
})
})
describe('down', () => {
beforeEach(async () => {
await queryInterface.createTable('sessions', {
id: { type: Sequelize.UUID, primaryKey: true },
lastRefreshToken: { type: Sequelize.STRING, allowNull: true },
lastRefreshTokenExpiresAt: { type: Sequelize.DATE, allowNull: true },
createdAt: { type: Sequelize.DATE, allowNull: false },
updatedAt: { type: Sequelize.DATE, allowNull: false }
})
})
it('should remove lastRefreshToken columns when they exist', async () => {
await down({ context: { queryInterface, logger: Logger } })
const tableDescription = await queryInterface.describeTable('sessions')
expect(tableDescription.lastRefreshToken).to.not.exist
expect(tableDescription.lastRefreshTokenExpiresAt).to.not.exist
})
it('should not fail when the columns do not exist', async () => {
await down({ context: { queryInterface, logger: Logger } })
await down({ context: { queryInterface, logger: Logger } })
expect(loggerInfoStub.calledWithMatch('lastRefreshToken column does not exist')).to.equal(true)
})
})
})

View file

@ -0,0 +1,70 @@
const { expect } = require('chai')
const sinon = require('sinon')
const Logger = require('../../../server/Logger')
const MediaProgress = require('../../../server/models/MediaProgress')
function createProgressInstance() {
const progress = Object.create(MediaProgress.prototype)
Object.defineProperties(progress, {
id: { value: 'progress-1', writable: true, configurable: true },
mediaItemId: { value: 'media-1', writable: true, configurable: true },
duration: { value: 3600, writable: true, configurable: true },
currentTime: { value: 120, writable: true, configurable: true },
isFinished: { value: false, writable: true, configurable: true },
hideFromContinueListening: { value: false, writable: true, configurable: true },
extraData: { value: {}, writable: true, configurable: true }
})
progress.changed = sinon.stub().returns(false)
progress.set = sinon.stub().callsFake((payload) => Object.assign(progress, payload))
progress.save = sinon.stub().resolves()
progress.reload = sinon.stub().resolves()
progress.constructor = {
update: sinon.stub().resolves(),
sequelize: {
escape: (value) => `'${value.toISOString ? value.toISOString() : value}'`
}
}
return progress
}
describe('MediaProgress', () => {
afterEach(() => {
sinon.restore()
})
describe('applyProgressUpdate', () => {
it('should update updatedAt via model update for valid lastUpdate', async () => {
const progress = createProgressInstance()
const infoStub = sinon.stub(Logger, 'info')
const lastUpdate = '2026-03-03T01:00:00.000Z'
await progress.applyProgressUpdate({ currentTime: 130, lastUpdate })
expect(progress.save.calledOnce).to.equal(true)
expect(progress.constructor.update.calledOnce).to.equal(true)
expect(progress.constructor.update.firstCall.args[0].updatedAt.toISOString()).to.equal(new Date(lastUpdate).toISOString())
expect(progress.constructor.update.firstCall.args[1]).to.deep.equal({
where: { id: 'progress-1' },
silent: true
})
expect(progress.reload.calledOnce).to.equal(true)
expect(infoStub.calledWithMatch('[MediaProgress] Manually setting updatedAt')).to.equal(true)
})
it('should skip manual updatedAt update when lastUpdate is invalid', async () => {
const progress = createProgressInstance()
const warnStub = sinon.stub(Logger, 'warn')
await progress.applyProgressUpdate({ currentTime: 130, lastUpdate: 'invalid-date' })
expect(progress.save.calledOnce).to.equal(true)
expect(progress.constructor.update.called).to.equal(false)
expect(progress.reload.called).to.equal(false)
expect(warnStub.calledWithMatch('[MediaProgress] Invalid date provided for lastUpdate')).to.equal(true)
})
})
})

View file

@ -0,0 +1,170 @@
const { expect } = require('chai')
const sinon = require('sinon')
const User = require('../../../server/models/User')
describe('User model', () => {
describe('case-insensitive lookup helpers', () => {
afterEach(() => {
sinon.restore()
})
it('should query usernames case-insensitively', async () => {
User.sequelize = {
models: {
mediaProgress: {}
}
}
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
await User.getUserByUsername('Madison')
expect(findOneStub.calledOnce).to.equal(true)
const options = findOneStub.firstCall.args[0]
expect(options.where.attribute.fn).to.equal('LOWER')
expect(options.where.attribute.args[0].col).to.equal('username')
expect(options.where.logic).to.equal('madison')
})
it('should query emails case-insensitively', async () => {
User.sequelize = {
models: {
mediaProgress: {}
}
}
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
await User.getUserByEmail('Example.User@Example.com')
expect(findOneStub.calledOnce).to.equal(true)
const options = findOneStub.firstCall.args[0]
expect(options.where.attribute.fn).to.equal('LOWER')
expect(options.where.attribute.args[0].col).to.equal('email')
expect(options.where.logic).to.equal('example.user@example.com')
})
it('should hit the user cache for mixed-case username lookups', async () => {
User.sequelize = {
models: {
mediaProgress: {}
}
}
const cachedUser = { id: 'cache-test-user-1', username: 'CacheTestUser', email: 'cachetest@example.com', extraData: {} }
const findOneStub = sinon.stub(User, 'findOne').resolves(cachedUser)
await User.getUserByUsername('CacheTestUser')
await User.getUserByUsername('cachetestuser')
await User.getUserByUsername('CACHETESTUSER')
expect(findOneStub.callCount).to.equal(1)
})
it('should hit the user cache for mixed-case email lookups', async () => {
User.sequelize = {
models: {
mediaProgress: {}
}
}
const cachedUser = { id: 'cache-test-user-2', username: 'CacheEmailUser', email: 'CacheMail@Example.com', extraData: {} }
const findOneStub = sinon.stub(User, 'findOne').resolves(cachedUser)
await User.getUserByEmail('CacheMail@Example.com')
await User.getUserByEmail('cachemail@example.com')
await User.getUserByEmail('CACHEMAIL@EXAMPLE.COM')
expect(findOneStub.callCount).to.equal(1)
})
})
describe('getUserByIdOrOldId', () => {
let originalSequelize
beforeEach(() => {
originalSequelize = User.sequelize
})
afterEach(() => {
User.sequelize = originalSequelize
sinon.restore()
})
it('should resolve UUID ids via primary-key lookup on postgres', async () => {
User.sequelize = {
getDialect: () => 'postgres',
models: {
mediaProgress: {}
}
}
const user = { id: 'e8e677b2-da16-4220-ab67-443b7714caf9' }
const findByPkStub = sinon.stub(User, 'findByPk').resolves(user)
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
const result = await User.getUserByIdOrOldId('e8e677b2-da16-4220-ab67-443b7714caf9')
expect(result).to.equal(user)
expect(findByPkStub.calledOnce).to.equal(true)
expect(findOneStub.called).to.equal(false)
})
it('should resolve uppercase UUID ids via primary-key lookup on postgres', async () => {
User.sequelize = {
getDialect: () => 'postgres',
models: {
mediaProgress: {}
}
}
const uppercaseUuid = 'E8E677B2-DA16-4220-AB67-443B7714CAF9'
const user = { id: uppercaseUuid }
const findByPkStub = sinon.stub(User, 'findByPk').resolves(user)
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
const result = await User.getUserByIdOrOldId(uppercaseUuid)
expect(result).to.equal(user)
expect(findByPkStub.calledOnceWithExactly(uppercaseUuid, { include: User.sequelize.models.mediaProgress })).to.equal(true)
expect(findOneStub.called).to.equal(false)
})
it('should query legacy oldUserId with postgres-safe JSON matcher', async () => {
User.sequelize = {
getDialect: () => 'postgres',
models: {
mediaProgress: {}
}
}
const findByPkStub = sinon.stub(User, 'findByPk').resolves(null)
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
await User.getUserByIdOrOldId('root')
expect(findByPkStub.called).to.equal(false)
expect(findOneStub.calledOnce).to.equal(true)
const options = findOneStub.firstCall.args[0]
expect(options.where.attribute.val).to.equal("extradata#>>'{oldUserId}'")
expect(options.where.logic).to.equal('root')
})
it('should keep sqlite oldUserId matcher unchanged', async () => {
User.sequelize = {
getDialect: () => 'sqlite',
models: {
mediaProgress: {}
}
}
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
await User.getUserByIdOrOldId('root')
expect(findOneStub.calledOnce).to.equal(true)
const options = findOneStub.firstCall.args[0]
expect(options.where).to.deep.equal({ 'extraData.oldUserId': 'root' })
})
})
})

View file

@ -0,0 +1,149 @@
const { expect } = require('chai')
const sqlite3 = require('sqlite3')
const {
normalizeJson,
isIntegerCompatible,
convertValue,
findOverlongVarcharValues,
findIntegerTypeIssues
} = require('../../../server/scripts/migrateSqliteToPostgres')
function openMemoryDb() {
return new Promise((resolve, reject) => {
const db = new sqlite3.Database(':memory:', (error) => {
if (error) return reject(error)
resolve(db)
})
})
}
function run(db, sql, params = []) {
return new Promise((resolve, reject) => {
db.run(sql, params, (error) => {
if (error) return reject(error)
resolve()
})
})
}
function close(db) {
return new Promise((resolve, reject) => {
db.close((error) => {
if (error) return reject(error)
resolve()
})
})
}
describe('migrateSqliteToPostgres script helpers', () => {
it('should keep malformed JSON payloads insertable by returning valid JSON text', () => {
const malformed = '"{"x",1}"'
const normalized = normalizeJson(malformed)
expect(() => JSON.parse(normalized)).to.not.throw()
expect(JSON.parse(normalized)).to.equal('"{"x",1}"')
})
it('should detect overlong varchar values before migration', async () => {
const db = await openMemoryDb()
try {
await run(db, 'CREATE TABLE books (subtitle TEXT)')
await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['x'.repeat(300)])
await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['ok'])
const tablesToMigrate = [{ sqliteTable: 'books', postgresTable: 'books' }]
const pgColumnsByTable = new Map([
[
'books',
new Map([
[
'subtitle',
{
column_name: 'subtitle',
data_type: 'character varying',
character_maximum_length: 255
}
]
])
]
])
const issues = await findOverlongVarcharValues(db, tablesToMigrate, pgColumnsByTable)
expect(issues).to.have.length(1)
expect(issues[0]).to.include({
sqliteTable: 'books',
sqliteColumn: 'subtitle',
postgresTable: 'books',
postgresColumn: 'subtitle',
maxLength: 255,
actualMaxLength: 300,
overCount: 1
})
} finally {
await close(db)
}
})
it('should detect non-integer values for integer target columns', async () => {
const db = await openMemoryDb()
try {
await run(db, 'CREATE TABLE playbackSessions (timeListening REAL)')
await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [25.802536999999997])
await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [42])
const tablesToMigrate = [{ sqliteTable: 'playbackSessions', postgresTable: 'playbackSessions' }]
const pgColumnsByTable = new Map([
[
'playbackSessions',
new Map([
[
'timelistening',
{
column_name: 'timeListening',
data_type: 'integer'
}
]
])
]
])
const issues = await findIntegerTypeIssues(db, tablesToMigrate, pgColumnsByTable)
expect(issues).to.have.length(1)
expect(issues[0].sqliteTable).to.equal('playbackSessions')
expect(issues[0].sqliteColumn).to.equal('timeListening')
expect(issues[0].postgresType).to.equal('integer')
expect(issues[0].badCount).to.equal(1)
} finally {
await close(db)
}
})
it('should only accept integer-compatible values for integer columns', () => {
expect(isIntegerCompatible(10)).to.equal(true)
expect(isIntegerCompatible('10')).to.equal(true)
expect(isIntegerCompatible(10.5)).to.equal(false)
expect(isIntegerCompatible('10.5')).to.equal(false)
expect(isIntegerCompatible('32768', 'smallint')).to.equal(false)
expect(isIntegerCompatible('9007199254740993', 'bigint')).to.equal(true)
})
it('should coerce integer-like strings for postgres integer columns', () => {
expect(convertValue('42', { data_type: 'integer' })).to.equal(42)
expect(convertValue('-7', { data_type: 'bigint' })).to.equal('-7')
expect(convertValue('9007199254740993', { data_type: 'bigint' })).to.equal('9007199254740993')
expect(convertValue('4.2', { data_type: 'integer' })).to.equal('4.2')
})
it('should always return valid json text for postgres json columns', () => {
const convertedObject = convertValue({ a: 1 }, { data_type: 'jsonb', udt_name: 'jsonb' })
const convertedMalformed = convertValue('"{"x",1}"', { data_type: 'jsonb', udt_name: 'jsonb' })
expect(JSON.parse(convertedObject)).to.deep.equal({ a: 1 })
expect(JSON.parse(convertedMalformed)).to.equal('"{"x",1}"')
})
})

View file

@ -0,0 +1,96 @@
const { expect } = require('chai')
const sinon = require('sinon')
const Database = require('../../../../server/Database')
const Logger = require('../../../../server/Logger')
const libraryFilters = require('../../../../server/utils/queries/libraryFilters')
const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters')
const libraryItemsPodcastFilters = require('../../../../server/utils/queries/libraryItemsPodcastFilters')
describe('libraryFilters shelf resilience', () => {
afterEach(() => {
sinon.restore()
})
it('should return empty discover shelf when discover query fails', async () => {
sinon.stub(Database, 'isPostgresDialect').returns(true)
sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').rejects(new Error('discover failed'))
const errorStub = sinon.stub(Logger, 'error')
const result = await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10)
expect(result).to.deep.equal({ libraryItems: [], count: 0 })
expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load discover shelf for library "library-1"')).to.equal(true)
})
it('should return empty in-progress shelf when a query fails', async () => {
sinon.stub(Database, 'isPostgresDialect').returns(true)
sinon.stub(libraryItemsBookFilters, 'getFilteredLibraryItems').rejects(new Error('progress failed'))
const errorStub = sinon.stub(Logger, 'error')
const result = await libraryFilters.getMediaItemsInProgress({ isBook: true, id: 'library-1' }, { id: 'user-1' }, [], 10)
expect(result).to.deep.equal({ items: [], count: 0 })
expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load in-progress shelf for library "library-1"')).to.equal(true)
})
it('should return empty continue-series shelf when a query fails', async () => {
sinon.stub(Database, 'isPostgresDialect').returns(true)
sinon.stub(libraryItemsBookFilters, 'getContinueSeriesLibraryItems').rejects(new Error('continue failed'))
const errorStub = sinon.stub(Logger, 'error')
const result = await libraryFilters.getLibraryItemsContinueSeries({ id: 'library-1' }, { id: 'user-1' }, [], 10)
expect(result).to.deep.equal({ libraryItems: [], count: 0 })
expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load continue-series shelf for library "library-1"')).to.equal(true)
})
it('should return empty newest podcast episodes shelf when a query fails', async () => {
sinon.stub(Database, 'isPostgresDialect').returns(true)
sinon.stub(libraryItemsPodcastFilters, 'getFilteredPodcastEpisodes').rejects(new Error('podcast failed'))
const errorStub = sinon.stub(Logger, 'error')
const result = await libraryFilters.getNewestPodcastEpisodes({ mediaType: 'podcast', id: 'library-1' }, { id: 'user-1' }, 10)
expect(result).to.deep.equal({ libraryItems: [], count: 0 })
expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load newest-podcast-episodes shelf for library "library-1"')).to.equal(true)
})
it('should rethrow shelf query errors on sqlite to keep upstream behavior', async () => {
sinon.stub(Database, 'isPostgresDialect').returns(false)
sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').rejects(new Error('discover failed'))
let error
try {
await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10)
} catch (caughtError) {
error = caughtError
}
expect(error).to.be.an('error')
expect(error.message).to.equal('discover failed')
})
it('should keep discover shelf mapping behavior when query succeeds', async () => {
const libraryItem = {
toOldJSONMinified: () => ({ id: 'item-1' }),
rssFeed: { toOldJSONMinified: () => ({ id: 'rss-1' }) },
mediaItemShare: { id: 'share-1' }
}
sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').resolves({
libraryItems: [libraryItem],
count: 1
})
const result = await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10)
expect(result.count).to.equal(1)
expect(result.libraryItems).to.deep.equal([
{
id: 'item-1',
rssFeed: { id: 'rss-1' },
mediaItemShare: { id: 'share-1' }
}
])
})
})

View file

@ -0,0 +1,148 @@
const { expect } = require('chai')
const sinon = require('sinon')
const Sequelize = require('sequelize')
const Database = require('../../../../server/Database')
const Logger = require('../../../../server/Logger')
const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters')
function createSequelizeStub(dialect, models) {
return {
getDialect: () => dialect,
escape: (value) => `'${String(value).replace(/'/g, "''")}'`,
random: () => ({ fn: 'random' }),
models
}
}
function createModelStubs() {
return {
book: {
findAndCountAll: sinon.stub(),
findAll: sinon.stub(),
count: sinon.stub()
},
libraryItem: {},
feed: {},
bookSeries: {},
series: {
findAll: sinon.stub()
},
bookAuthor: {},
author: {},
mediaProgress: {},
mediaItemShare: {}
}
}
describe('libraryItemsBookFilters postgres query safety', () => {
let originalSequelize
let originalServerSettings
let modelStubs
beforeEach(() => {
originalSequelize = Database.sequelize
originalServerSettings = global.ServerSettings
modelStubs = createModelStubs()
global.ServerSettings = {
sortingIgnorePrefix: false
}
})
afterEach(() => {
Database.sequelize = originalSequelize
global.ServerSettings = originalServerSettings
sinon.restore()
})
it('should avoid generating IN () when collapse-series has no include ids', async () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 })
sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({
booksToExclude: [],
bookSeriesToInclude: []
})
const debugStub = sinon.stub(Logger, 'debug')
await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, true, [], 20, 0)
const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0]
const displayTitleExpression = findOptions.attributes.include[0][0].val
expect(displayTitleExpression).to.include('COALESCE(NULL, libraryItem.title)')
expect(displayTitleExpression).to.not.include('IN ()')
expect(debugStub.calledWithMatch('collapse-series produced no include IDs')).to.equal(true)
})
it('should escape collapse-series ids safely for postgres subquery', async () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 })
sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({
booksToExclude: [],
bookSeriesToInclude: [{ id: "series-id-'quoted'", numBooks: 2, libraryItemIds: [] }]
})
global.ServerSettings.sortingIgnorePrefix = true
await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.title', false, true, [], 20, 0)
const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0]
const displayTitleExpression = findOptions.attributes.include[0][0].val
expect(displayTitleExpression).to.include("bs.id IN ('series-id-''quoted''')")
expect(displayTitleExpression).to.include('libraryItem.titleIgnorePrefix')
})
it('should use postgres-safe join alias for sequence sorting', () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
const order = libraryItemsBookFilters.getOrder('sequence', false, false)
const expression = order[0][0].val
expect(expression).to.include('"series->bookSeries"."sequence"')
expect(expression).to.include('CASE WHEN BTRIM')
})
it('should log generated query when findAndCountAll fails', async () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
modelStubs.book.findAndCountAll.rejects(new Error('boom'))
const errorStub = sinon.stub(Logger, 'error')
try {
await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, false, [], 20, 0)
expect.fail('Expected getFilteredLibraryItems to throw')
} catch (error) {
expect(error.message).to.equal('boom')
}
expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll failed: boom')).to.equal(true)
expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll query:')).to.equal(true)
})
it('should use postgres-safe books->bookSeries alias in collapse-series selector query', async () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
modelStubs.series.findAll.resolves([])
await libraryItemsBookFilters.getCollapseSeriesBooksToExclude({ where: {}, include: [] }, null)
const findAllOptions = modelStubs.series.findAll.firstCall.args[0]
const orderExpression = findAllOptions.order[0].val
expect(orderExpression).to.include('"books->bookSeries"."sequence"')
})
it('should use boolean false for discover not-started media progress on postgres', async () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
modelStubs.series.findAll.resolves([])
modelStubs.book.count.resolves(0)
modelStubs.book.findAll.resolves([])
await libraryItemsBookFilters.getDiscoverLibraryItems('library-1', { id: 'user-1', canAccessExplicitContent: true }, [], 10)
const countOptions = modelStubs.book.count.firstCall.args[0]
const progressFilter = countOptions.where[0]['$mediaProgresses.isFinished$']
expect(progressFilter[Sequelize.Op.or]).to.deep.equal([null, false])
})
})

View file

@ -0,0 +1,75 @@
const { expect } = require('chai')
const sinon = require('sinon')
const Database = require('../../../../server/Database')
const libraryItemsPodcastFilters = require('../../../../server/utils/queries/libraryItemsPodcastFilters')
describe('libraryItemsPodcastFilters dialect behavior', () => {
let originalSequelize
let originalServerSettings
beforeEach(() => {
originalSequelize = Database.sequelize
originalServerSettings = global.ServerSettings
global.ServerSettings = { sortingIgnorePrefix: false }
})
afterEach(() => {
Database.sequelize = originalSequelize
global.ServerSettings = originalServerSettings
sinon.restore()
})
it('should build postgres json tag permission predicate', () => {
Database.sequelize = { getDialect: () => 'postgres' }
const { podcastWhere, replacements } = libraryItemsPodcastFilters.getUserPermissionPodcastWhereQuery({
canAccessExplicitContent: true,
permissions: {
accessAllTags: false,
itemTagsSelected: ['fiction'],
selectedTagsNotAccessible: false
}
})
expect(replacements.userTagsSelected).to.deep.equal(['fiction'])
expect(podcastWhere[0].attribute.val).to.include('jsonb_array_elements_text')
})
it('should build sqlite no-case author sort expression', () => {
Database.sequelize = { getDialect: () => 'sqlite' }
const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false)
expect(order[0][0].val).to.include('podcast.author COLLATE NOCASE')
})
it('should build postgres lower author sort expression', () => {
Database.sequelize = { getDialect: () => 'postgres' }
const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false)
expect(order[0][0].val).to.include('LOWER(podcast.author)')
})
it('should use postgres json duration extraction in podcast stats query', async () => {
const queryStub = sinon.stub()
queryStub.onFirstCall().resolves([[{ totalSize: 1000 }]])
queryStub.onSecondCall().resolves([[{ totalDuration: '12.3', totalItems: '4', numAudioFiles: '9' }]])
Database.sequelize = {
getDialect: () => 'postgres',
query: queryStub
}
const result = await libraryItemsPodcastFilters.getPodcastLibraryStats('library-1')
expect(queryStub.secondCall.args[0]).to.include('NULLIF(pe.audioFile::jsonb #>>')
expect(result).to.deep.equal({
totalSize: 1000,
totalDuration: '12.3',
numAudioFiles: '9',
totalItems: '4'
})
})
})

View file

@ -0,0 +1,107 @@
const { expect } = require('chai')
const sinon = require('sinon')
const Database = require('../../../../server/Database')
const seriesFilters = require('../../../../server/utils/queries/seriesFilters')
const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters')
function encodedFilter(group, value) {
return `${group}.${encodeURIComponent(Buffer.from(value).toString('base64'))}`
}
describe('seriesFilters dialect behavior', () => {
let originalSequelize
let originalServerSettings
let modelStubs
function setDialect(dialect) {
Database.sequelize = {
getDialect: () => dialect,
random: sinon.stub(),
models: modelStubs
}
}
beforeEach(() => {
originalSequelize = Database.sequelize
originalServerSettings = global.ServerSettings
modelStubs = {
series: {
findAndCountAll: sinon.stub().resolves({ rows: [], count: 0 })
},
bookSeries: {},
book: {},
libraryItem: {},
author: {},
feed: {}
}
global.ServerSettings = { sortingIgnorePrefix: false }
sinon.stub(libraryItemsBookFilters, 'getUserPermissionBookWhereQuery').returns({
bookWhere: [],
replacements: {}
})
})
afterEach(() => {
Database.sequelize = originalSequelize
global.ServerSettings = originalServerSettings
sinon.restore()
})
it('should build postgres progress filter with TRUE and FALSE literals', async () => {
setDialect('postgres')
await seriesFilters.getFilteredSeries(
{ id: 'library-1', settings: { hideSingleBookSeries: false } },
{ id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } },
encodedFilter('progress', 'not-started'),
'name',
false,
[],
10,
0
)
const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0]
const progressWhere = findOptions.where[1]
expect(progressWhere.attribute.val).to.include('mp.isFinished = TRUE')
})
it('should build sqlite no-case name sort expression', async () => {
setDialect('sqlite')
await seriesFilters.getFilteredSeries(
{ id: 'library-1', settings: { hideSingleBookSeries: false } },
{ id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } },
null,
'name',
false,
[],
10,
0
)
const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0]
expect(findOptions.order[0][0].val).to.equal('series.name COLLATE NOCASE')
})
it('should build postgres lower-case name sort expression', async () => {
setDialect('postgres')
await seriesFilters.getFilteredSeries(
{ id: 'library-1', settings: { hideSingleBookSeries: false } },
{ id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } },
null,
'name',
false,
[],
10,
0
)
const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0]
expect(findOptions.order[0][0].val).to.equal('LOWER(series.name)')
})
})

View file

@ -0,0 +1,47 @@
const { expect } = require('chai')
const fs = require('fs')
const path = require('path')
function getJsFiles(dirPath) {
const entries = fs.readdirSync(dirPath, { withFileTypes: true })
return entries.flatMap((entry) => {
const entryPath = path.join(dirPath, entry.name)
if (entry.isDirectory()) return getJsFiles(entryPath)
if (entry.isFile() && entry.name.endsWith('.js')) return [entryPath]
return []
})
}
describe('query boolean safety', () => {
it('should not use numeric literals for SQL boolean fields in query code', () => {
const queriesDir = path.resolve(__dirname, '../../../../server/utils/queries')
const files = getJsFiles(queriesDir)
const offenders = []
const forbiddenPatterns = [
{
regex: /\$mediaProgresses\.isFinished\$[\s\S]{0,120}\[null,\s*[01]\]/g,
message: 'numeric literal in mediaProgresses.isFinished filter'
},
{
regex: /\$books\.mediaProgresses\.isFinished\$[\s\S]{0,120}\[null,\s*[01]\]/g,
message: 'numeric literal in books.mediaProgresses.isFinished filter'
},
{
regex: /\b(?:mp|mediaProgresses|b)\.(?:isFinished|explicit|abridged)\s*=\s*[01]\b/g,
message: 'direct SQL boolean comparison against 0/1'
}
]
files.forEach((filePath) => {
const content = fs.readFileSync(filePath, 'utf8')
forbiddenPatterns.forEach(({ regex, message }) => {
const matches = [...content.matchAll(regex)]
matches.forEach((match) => {
offenders.push(`${path.relative(queriesDir, filePath)}: ${message}: ${match[0].replace(/\s+/g, ' ').trim()}`)
})
})
})
expect(offenders).to.deep.equal([])
})
})

View file

@ -0,0 +1,68 @@
const { expect } = require('chai')
const {
booleanLiteral,
noCaseSortExpression,
coalesceFunctionName,
jsonArrayContainsAny,
jsonArrayContainsValue,
jsonArrayExpand,
jsonPathText,
jsonPathNumber,
safeTextToDoubleExpression,
safeTextToIntegerExpression
} = require('../../../server/utils/sqlDialectHelpers')
const sqlite = {
getDialect: () => 'sqlite'
}
const postgres = {
getDialect: () => 'postgres'
}
describe('sqlDialectHelpers', () => {
it('should return sqlite and postgres boolean literals', () => {
expect(booleanLiteral(true, sqlite)).to.equal('1')
expect(booleanLiteral(false, sqlite)).to.equal('0')
expect(booleanLiteral(true, postgres)).to.equal('TRUE')
expect(booleanLiteral(false, postgres)).to.equal('FALSE')
})
it('should generate case-insensitive sort expressions', () => {
expect(noCaseSortExpression('name', sqlite)).to.equal('name COLLATE NOCASE')
expect(noCaseSortExpression('name', postgres)).to.equal('LOWER(name)')
})
it('should choose the correct null-coalescing function name', () => {
expect(coalesceFunctionName(sqlite)).to.equal('IFNULL')
expect(coalesceFunctionName(postgres)).to.equal('COALESCE')
})
it('should generate array membership count queries per dialect', () => {
expect(jsonArrayContainsAny('tags', 'selectedTags', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:selectedTags))')
expect(jsonArrayContainsAny('tags', 'selectedTags', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:selectedTags))")
expect(jsonArrayContainsValue('tags', 'tag', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value = :tag)')
expect(jsonArrayContainsValue('tags', 'tag', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :tag)")
})
it('should generate json array expansion and path extraction by dialect', () => {
expect(jsonArrayExpand('books.tags', sqlite)).to.equal('json_each(books.tags)')
expect(jsonArrayExpand('books.tags', postgres)).to.equal("jsonb_array_elements_text(COALESCE(books.tags::jsonb, '[]'::jsonb)) AS json_each(value)")
expect(jsonPathText('payload', ['metadata', 'filename'], sqlite)).to.equal("json_extract(payload, '$.metadata.filename')")
expect(jsonPathText('payload', ['metadata', 'filename'], postgres)).to.equal("payload::jsonb #>> '{metadata,filename}'")
expect(jsonPathNumber('payload', ['duration'], sqlite)).to.equal("json_extract(payload, '$.duration')")
expect(jsonPathNumber('payload', ['duration'], postgres)).to.equal("NULLIF(payload::jsonb #>> '{duration}', '')::double precision")
})
it('should generate safe numeric cast expressions for sequence-like text', () => {
expect(safeTextToDoubleExpression('sequence', sqlite)).to.equal('CAST(sequence AS FLOAT)')
expect(safeTextToDoubleExpression('"bookSeries"."sequence"', postgres)).to.equal("CASE WHEN BTRIM(\"bookSeries\".\"sequence\") ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(\"bookSeries\".\"sequence\")::double precision ELSE NULL END")
expect(safeTextToIntegerExpression('publishedYear', sqlite)).to.equal('CAST(publishedYear AS INTEGER)')
expect(safeTextToIntegerExpression('book.publishedYear', postgres)).to.equal("CASE WHEN BTRIM(book.publishedYear) ~ '^[+-]?\\d+$' THEN BTRIM(book.publishedYear)::integer ELSE NULL END")
})
})