groove/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2026-08-02 22:51:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge 8c97a35206 into 6caa1aa2da

Browse source
This commit is contained in:
Kevin GATERA 2026-08-02 22:41:27 +00:00 • committed by GitHub
commit cdfff4dc8b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
57 changed files with 3837 additions and 508 deletions
Download patch file Download diff file Expand all files Collapse all files

45
.github/workflows/homelab-docker-build.yml vendored Normal file
View file

@ -0,0 +1,45 @@
name: Homelab Docker Build
on:
push:
branches:
- homelab
- homelab-postgres
workflow_dispatch:
permissions:
contents: read
packages: write
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: Check out
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push homelab image
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64
push: true
tags: |
ghcr.io/kevingatera/audiobookshelf:${{ github.ref_name }}
ghcr.io/kevingatera/audiobookshelf:${{ github.ref_name }}-${{ github.sha }}
labels: |
org.opencontainers.image.source=https://github.com/kevingatera/audiobookshelf
org.opencontainers.image.revision=${{ github.sha }}

1
Dockerfile
View file

@ -50,6 +50,7 @@ ARG NUSQLITE3_PATH
RUN apk add --no-cache --update \
tzdata \
ffmpeg \
postgresql-client \
tini
WORKDIR /app

167
package-lock.json generated
View file

@ -23,6 +23,8 @@
"p-throttle": "^4.1.1",
"passport": "^0.6.0",
"passport-jwt": "^4.0.1",
"pg": "^8.19.0",
"pg-hstore": "^2.3.4",
"semver": "^7.6.3",
"sequelize": "^6.35.2",
"socket.io": "^4.5.4",
@ -4138,10 +4140,106 @@
"resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz",
"integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg=="
},
"node_modules/pg": {
"version": "8.22.0",
"resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz",
"integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==",
"license": "MIT",
"dependencies": {
"pg-connection-string": "^2.14.0",
"pg-pool": "^3.14.0",
"pg-protocol": "^1.15.0",
"pg-types": "2.2.0",
"pgpass": "1.0.5"
},
"engines": {
"node": ">= 16.0.0"
},
"optionalDependencies": {
"pg-cloudflare": "^1.4.0"
},
"peerDependencies": {
"pg-native": ">=3.0.1"
},
"peerDependenciesMeta": {
"pg-native": {
"optional": true
}
}
},
"node_modules/pg-cloudflare": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz",
"integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==",
"license": "MIT",
"optional": true
},
"node_modules/pg-connection-string": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz",
"integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg=="
"version": "2.14.0",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz",
"integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==",
"license": "MIT"
},
"node_modules/pg-hstore": {
"version": "2.3.4",
"resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz",
"integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==",
"license": "MIT",
"dependencies": {
"underscore": "^1.13.1"
},
"engines": {
"node": ">= 0.8.x"
}
},
"node_modules/pg-int8": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
"license": "ISC",
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/pg-pool": {
"version": "3.14.0",
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
"integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
"license": "MIT",
"peerDependencies": {
"pg": ">=8.0"
}
},
"node_modules/pg-protocol": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz",
"integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==",
"license": "MIT"
},
"node_modules/pg-types": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
"license": "MIT",
"dependencies": {
"pg-int8": "1.0.1",
"postgres-array": "~2.0.0",
"postgres-bytea": "~1.0.0",
"postgres-date": "~1.0.4",
"postgres-interval": "^1.1.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/pgpass": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
"license": "MIT",
"dependencies": {
"split2": "^4.1.0"
}
},
"node_modules/picocolors": {
"version": "1.0.0",
@ -4173,6 +4271,45 @@
"node": ">=8"
}
},
"node_modules/postgres-array": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/postgres-bytea": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
"integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/postgres-date": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/postgres-interval": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
"license": "MIT",
"dependencies": {
"xtend": "^4.0.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/prebuild-install": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
@ -4946,6 +5083,15 @@
"node": ">=8"
}
},
"node_modules/split2": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
"license": "ISC",
"engines": {
"node": ">= 10.x"
}
},
"node_modules/sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@ -5265,6 +5411,12 @@
"integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
"dev": true
},
"node_modules/underscore": {
"version": "1.13.8",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz",
"integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==",
"license": "MIT"
},
"node_modules/unique-filename": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz",
@ -5476,6 +5628,15 @@
"node": ">=4.0"
}
},
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
"license": "MIT",
"engines": {
"node": ">=0.4"
}
},
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

3
package.json
View file

@ -16,6 +16,7 @@
"docker": "docker buildx build --platform linux/amd64,linux/arm64 --push . -t advplyr/audiobookshelf",
"docker-amd64-local": "docker buildx build --platform linux/amd64 --load . -t advplyr/audiobookshelf-amd64-local",
"docker-arm64-local": "docker buildx build --platform linux/arm64 --load . -t advplyr/audiobookshelf-arm64-local",
"migrate-sqlite-to-postgres": "node server/scripts/migrateSqliteToPostgres.js",
"deploy-linux": "node deploy/linux",
"test": "mocha",
"coverage": "nyc mocha"
@ -52,6 +53,8 @@
"p-throttle": "^4.1.1",
"passport": "^0.6.0",
"passport-jwt": "^4.0.1",
"pg": "^8.19.0",
"pg-hstore": "^2.3.4",
"semver": "^7.6.3",
"sequelize": "^6.35.2",
"socket.io": "^4.5.4",

290
server/Database.js
View file

@ -14,6 +14,7 @@ class Database {
constructor() {
this.sequelize = null
this.dbPath = null
this.dialect = 'sqlite'
this.isNew = false // New absdatabase.sqlite created
this.hasRootUser = false // Used to show initialization page in web ui
@ -162,11 +163,37 @@ class Database {
return this.models.device
}
/**
* @returns {'sqlite'|'postgres'}
*/
getConfiguredDialect() {
const explicitDialect = process.env.DB_DIALECT?.trim()?.toLowerCase()
if (explicitDialect === 'postgres' || explicitDialect === 'sqlite') {
return explicitDialect
}
const databaseUrl = process.env.DATABASE_URL || ''
if (databaseUrl.startsWith('postgres://') || databaseUrl.startsWith('postgresql://')) {
return 'postgres'
}
return 'sqlite'
}
isSqliteDialect() {
return this.dialect === 'sqlite'
}
isPostgresDialect() {
return this.dialect === 'postgres'
}
/**
* Check if db file exists
* @returns {boolean}
*/
async checkHasDb() {
if (!this.isSqliteDialect()) return true
if (!(await fs.pathExists(this.dbPath))) {
Logger.info(`[Database] absdatabase.sqlite not found at ${this.dbPath}`)
return false
@ -174,20 +201,39 @@ class Database {
return true
}
/**
* Check if any user tables exist (for networked dialects)
* @returns {Promise<boolean>}
*/
async checkHasTables() {
const queryInterface = this.sequelize.getQueryInterface()
const tables = await queryInterface.showAllTables()
return Array.isArray(tables) && tables.length > 0
}
/**
* Connect to db, build models and run migrations
* @param {boolean} [force=false] Used for testing, drops & re-creates all tables
*/
async init(force = false) {
this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite')
// First check if this is a new database
this.isNew = !(await this.checkHasDb()) || force
this.dialect = this.getConfiguredDialect()
if (this.isSqliteDialect()) {
this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite')
// First check if this is a new database
this.isNew = !(await this.checkHasDb()) || force
} else {
this.dbPath = process.env.DATABASE_URL || null
this.isNew = !!force
}
if (!(await this.connect())) {
throw new Error('Database connection failed')
}
if (this.isPostgresDialect() && !force) {
this.isNew = !(await this.checkHasTables())
}
try {
const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath)
await migrationManager.init(packageJson.version)
@ -214,7 +260,11 @@ class Database {
* @returns {boolean}
*/
async connect() {
Logger.info(`[Database] Initializing db at "${this.dbPath}"`)
if (this.isSqliteDialect()) {
Logger.info(`[Database] Initializing sqlite db at "${this.dbPath}"`)
} else {
Logger.info(`[Database] Initializing postgres db connection`)
}
let logging = false
let benchmark = false
@ -229,13 +279,27 @@ class Database {
benchmark = true
}
this.sequelize = new Sequelize({
dialect: 'sqlite',
storage: this.dbPath,
logging: logging,
benchmark: benchmark,
transactionType: 'IMMEDIATE'
})
if (this.isSqliteDialect()) {
this.sequelize = new Sequelize({
dialect: 'sqlite',
storage: this.dbPath,
logging: logging,
benchmark: benchmark,
transactionType: 'IMMEDIATE'
})
} else {
if (!process.env.DATABASE_URL) {
Logger.error(`[Database] DATABASE_URL is required when DB_DIALECT=postgres`)
return false
}
this.sequelize = new Sequelize(process.env.DATABASE_URL, {
dialect: 'postgres',
logging: logging,
benchmark: benchmark,
quoteIdentifiers: false
})
}
// Helper function
this.sequelize.uppercaseFirst = (str) => (str ? `${str[0].toUpperCase()}${str.substr(1)}` : '')
@ -243,34 +307,36 @@ class Database {
try {
await this.sequelize.authenticate()
// Set SQLite pragmas from environment variables
const allowedPragmas = [
{ name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' },
{ name: 'cache_size', env: 'SQLITE_CACHE_SIZE' },
{ name: 'temp_store', env: 'SQLITE_TEMP_STORE' }
]
if (this.isSqliteDialect()) {
// Set SQLite pragmas from environment variables
const allowedPragmas = [
{ name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' },
{ name: 'cache_size', env: 'SQLITE_CACHE_SIZE' },
{ name: 'temp_store', env: 'SQLITE_TEMP_STORE' }
]
for (const pragma of allowedPragmas) {
const value = process.env[pragma.env]
if (value !== undefined) {
try {
Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`)
await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`)
const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`)
Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result)
} catch (error) {
Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error)
for (const pragma of allowedPragmas) {
const value = process.env[pragma.env]
if (value !== undefined) {
try {
Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`)
await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`)
const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`)
Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result)
} catch (error) {
Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error)
}
}
}
}
if (process.env.NUSQLITE3_PATH) {
await this.loadExtension(process.env.NUSQLITE3_PATH)
Logger.info(`[Database] Db supports unaccent and unicode foldings`)
this.supportsUnaccent = true
this.supportsUnicodeFoldings = true
if (process.env.NUSQLITE3_PATH) {
await this.loadExtension(process.env.NUSQLITE3_PATH)
Logger.info(`[Database] Db supports unaccent and unicode foldings`)
this.supportsUnaccent = true
this.supportsUnicodeFoldings = true
}
}
Logger.info(`[Database] Db connection was successful`)
Logger.info(`[Database] Db connection was successful (${this.dialect})`)
return true
} catch (error) {
Logger.error(`[Database] Failed to connect to db`, error)
@ -307,7 +373,7 @@ class Database {
* Disconnect from db
*/
async disconnect() {
Logger.info(`[Database] Disconnecting sqlite db`)
Logger.info(`[Database] Disconnecting ${this.dialect} db`)
await this.sequelize.close()
}
@ -315,7 +381,7 @@ class Database {
* Reconnect to db and init
*/
async reconnect() {
Logger.info(`[Database] Reconnecting sqlite db`)
Logger.info(`[Database] Reconnecting ${this.dialect} db`)
await this.init()
}
@ -346,6 +412,11 @@ class Database {
require('./models/CustomMetadataProvider').init(this.sequelize)
require('./models/MediaItemShare').init(this.sequelize)
if (this.isPostgresDialect() && !force && !this.isNew) {
Logger.info('[Database] Skipping sequelize.sync for existing postgres schema')
return Promise.resolve()
}
return this.sequelize.sync({ force, alter: false })
}
@ -368,7 +439,7 @@ class Database {
* Loads most of the data from the database. This is a temporary solution.
*/
async loadData() {
if (this.isNew && (await dbMigration.checkShouldMigrate())) {
if (this.isSqliteDialect() && this.isNew && (await dbMigration.checkShouldMigrate())) {
Logger.info(`[Database] New database was created and old database was detected - migrating old to new`)
await dbMigration.migrate(this.models)
}
@ -441,9 +512,19 @@ class Database {
return this.models.setting.updateSettingObj(settings.toJSON())
}
getPlaybackSessions(where = null) {
getPlaybackSessions(where = null, options = null) {
if (!this.sequelize) return false
return this.models.playbackSession.getOldPlaybackSessions(where)
return this.models.playbackSession.getOldPlaybackSessions(where, options || undefined)
}
countPlaybackSessions(where = null) {
if (!this.sequelize) return false
return this.models.playbackSession.countWithWhere(where)
}
getPlaybackSessionsForStats(where = null) {
if (!this.sequelize) return false
return this.models.playbackSession.getPlaybackSessionsForStats(where)
}
getPlaybackSession(sessionId) {
@ -847,6 +928,14 @@ WHERE EXISTS (
* It adds triggers to update libraryItems.title[IgnorePrefix] when (books|podcasts).title[IgnorePrefix] is updated
*/
async addTriggers() {
if (this.isPostgresDialect()) {
return this.addPostgresTriggers()
}
if (!this.isSqliteDialect()) {
Logger.info(`[Database] Skipping sqlite-only triggers for dialect ${this.dialect}`)
return
}
await this.addTriggerIfNotExists('books', 'title', 'id', 'libraryItems', 'title', 'mediaId')
await this.addTriggerIfNotExists('books', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId')
await this.addTriggerIfNotExists('podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId')
@ -944,6 +1033,121 @@ WHERE EXISTS (
await addAuthorsUpdateTriggerIfNotExists()
}
/**
* Postgres equivalent of the sqlite libraryItems denormalization triggers above.
* Identifiers are unquoted (and therefore folded to lowercase by postgres) to match
* the quoteIdentifiers: false strategy used for the postgres dialect.
*/
async addPostgresTriggers() {
Logger.info('[Database] Adding postgres denormalization triggers')
await this.addPostgresTitleTriggerIfNotExists('books', 'title')
await this.addPostgresTitleTriggerIfNotExists('books', 'titleIgnorePrefix')
await this.addPostgresTitleTriggerIfNotExists('podcasts', 'title')
await this.addPostgresTitleTriggerIfNotExists('podcasts', 'titleIgnorePrefix')
await this.addPostgresAuthorNamesTriggersIfNotExist()
}
async postgresTriggerExists(triggerName) {
const [[{ count }]] = await this.sequelize.query(`SELECT COUNT(*) as count FROM pg_trigger WHERE NOT tgisinternal AND tgname = '${triggerName}'`)
return Number(count) > 0
}
async addPostgresTitleTriggerIfNotExists(sourceTable, sourceColumn) {
const foldedColumn = sourceColumn.toLowerCase()
const action = `update_libraryItems_${sourceColumn}`
const fromSource = sourceTable === 'books' ? '' : `_from_${sourceTable}_${sourceColumn}`
const triggerName = this.convertToSnakeCase(`${action}${fromSource}`)
const functionName = `${triggerName}_fn`
if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists
Logger.info(`[Database] Adding trigger ${triggerName}`)
await this.sequelize.query(`
CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$
BEGIN
UPDATE libraryitems
SET ${foldedColumn} = NEW.${foldedColumn}
WHERE mediaid = NEW.id;
RETURN NEW;
END;
$func$ LANGUAGE plpgsql
`)
await this.sequelize.query(`
CREATE TRIGGER ${triggerName}
AFTER UPDATE OF ${foldedColumn} ON ${sourceTable}
FOR EACH ROW
EXECUTE FUNCTION ${functionName}()
`)
}
async addPostgresAuthorNamesTriggersIfNotExist() {
// string_agg is the postgres equivalent of sqlite GROUP_CONCAT; both return NULL for an empty set
const authorNamesSubQuery = (bookIdExpression) => `
SELECT string_agg(authors.name, ', ' ORDER BY bookauthors.createdat ASC), string_agg(authors.lastfirst, ', ' ORDER BY bookauthors.createdat ASC)
FROM authors JOIN bookauthors ON authors.id = bookauthors.authorid
WHERE bookauthors.bookid = ${bookIdExpression}
`
const addBookAuthorsTriggerIfNotExists = async (action) => {
const modifiedRecord = action === 'delete' ? 'OLD' : 'NEW'
const triggerName = this.convertToSnakeCase(`update_libraryItems_authorNames_on_bookAuthors_${action}`)
const functionName = `${triggerName}_fn`
if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists
Logger.info(`[Database] Adding trigger ${triggerName}`)
await this.sequelize.query(`
CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$
BEGIN
UPDATE libraryitems
SET (authornamesfirstlast, authornameslastfirst) = (${authorNamesSubQuery(`${modifiedRecord}.bookid`)})
WHERE mediaid = ${modifiedRecord}.bookid;
RETURN ${modifiedRecord};
END;
$func$ LANGUAGE plpgsql
`)
await this.sequelize.query(`
CREATE TRIGGER ${triggerName}
AFTER ${action.toUpperCase()} ON bookauthors
FOR EACH ROW
EXECUTE FUNCTION ${functionName}()
`)
}
const addAuthorsUpdateTriggerIfNotExists = async () => {
const triggerName = this.convertToSnakeCase('update_libraryItems_authorNames_on_authors_update')
const functionName = `${triggerName}_fn`
if (await this.postgresTriggerExists(triggerName)) return // Trigger already exists
Logger.info(`[Database] Adding trigger ${triggerName}`)
await this.sequelize.query(`
CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $func$
BEGIN
UPDATE libraryitems
SET (authornamesfirstlast, authornameslastfirst) = (${authorNamesSubQuery('libraryitems.mediaid')})
WHERE mediaid IN (SELECT bookid FROM bookauthors WHERE authorid = NEW.id);
RETURN NEW;
END;
$func$ LANGUAGE plpgsql
`)
await this.sequelize.query(`
CREATE TRIGGER ${triggerName}
AFTER UPDATE OF name ON authors
FOR EACH ROW
EXECUTE FUNCTION ${functionName}()
`)
}
await addBookAuthorsTriggerIfNotExists('insert')
await addBookAuthorsTriggerIfNotExists('delete')
await addAuthorsUpdateTriggerIfNotExists()
}
convertToSnakeCase(str) {
return str.replace(/([A-Z])/g, '_$1').toLowerCase()
}
@ -954,6 +1158,7 @@ WHERE EXISTS (
this.supportsUnaccent = supportsUnaccent
this.query = query
this.hasAccents = false
this.dialect = sequelize.getDialect()
}
/**
@ -989,9 +1194,10 @@ WHERE EXISTS (
*/
matchExpression(column) {
const pattern = this.sequelize.escape(`%${this.query}%`)
if (!this.supportsUnaccent) return `${column} LIKE ${pattern}`
const likeOperator = this.dialect === 'postgres' ? 'ILIKE' : 'LIKE'
if (!this.supportsUnaccent) return `${column} ${likeOperator} ${pattern}`
const normalizedColumn = this.hasAccents ? column : this.normalize(column)
return `${normalizedColumn} LIKE ${pattern}`
return `${normalizedColumn} ${likeOperator} ${pattern}`
}
}
}

25
server/Server.js
View file

@ -479,14 +479,27 @@ class Server {
// Remove series from hide from continue listening that no longer exist
try {
const users = await Database.sequelize.query(`SELECT u.id, u.username, u.extraData, json_group_array(value) AS seriesIdsToRemove FROM users u, json_each(u.extraData->"seriesHideFromContinueListening") LEFT JOIN series se ON se.id = value WHERE se.id IS NULL GROUP BY u.id;`, {
model: Database.userModel,
type: Sequelize.QueryTypes.SELECT
const users = await Database.userModel.findAll({
attributes: ['id', 'username', 'extraData']
})
for (const user of users) {
const extraData = JSON.parse(user.extraData)
const existingSeriesIds = extraData.seriesHideFromContinueListening
const seriesIdsToRemove = JSON.parse(user.dataValues.seriesIdsToRemove)
const extraData = typeof user.extraData === 'string' ? JSON.parse(user.extraData || '{}') : user.extraData || {}
const existingSeriesIds = Array.isArray(extraData.seriesHideFromContinueListening) ? extraData.seriesHideFromContinueListening : []
if (!existingSeriesIds.length) continue
const existingSeries = await Database.seriesModel.findAll({
attributes: ['id'],
where: {
id: {
[Sequelize.Op.in]: existingSeriesIds
}
}
})
const existingSeriesSet = new Set(existingSeries.map((series) => series.id))
const seriesIdsToRemove = existingSeriesIds.filter((seriesId) => !existingSeriesSet.has(seriesId))
if (!seriesIdsToRemove.length) continue
Logger.info(`[Server] Found ${seriesIdsToRemove.length} non-existent series in seriesHideFromContinueListening for user "${user.username}" - Removing (${seriesIdsToRemove.join(',')})`)
const newExtraData = {
...extraData,

16
server/controllers/LibraryController.js
View file

@ -24,6 +24,7 @@ const libraryFilters = require('../utils/queries/libraryFilters')
const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters')
const authorFilters = require('../utils/queries/authorFilters')
const zipHelpers = require('../utils/zipHelpers')
const { isPostgres, noCaseSortExpression, jsonArrayExpand } = require('../utils/sqlDialectHelpers')
/**
* @typedef RequestUserObject
@ -1039,9 +1040,9 @@ class LibraryController {
let order = undefined
const direction = payload.sortDesc ? 'DESC' : 'ASC'
if (payload.sortBy === 'name') {
order = [[Sequelize.literal('name COLLATE NOCASE'), direction]]
order = [[Sequelize.literal(noCaseSortExpression('name', Database.sequelize)), direction]]
} else if (payload.sortBy === 'lastFirst') {
order = [[Sequelize.literal('lastFirst COLLATE NOCASE'), direction]]
order = [[Sequelize.literal(noCaseSortExpression('lastFirst', Database.sequelize)), direction]]
} else if (payload.sortBy === 'addedAt') {
order = [['createdAt', direction]]
} else if (payload.sortBy === 'updatedAt') {
@ -1338,16 +1339,23 @@ class LibraryController {
const fileExt = req.query.ext === 'abs' ? 'abs' : 'json'
const metadataFilename = `metadata.${fileExt}`
const metadataFilenameCountQuery = isPostgres(Database.sequelize)
? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{metadata,filename}' = :metadataFilename)`
: `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = :metadataFilename)`
const libraryItemsWithMetadata = await Database.libraryItemModel.findAll({
attributes: ['id', 'libraryFiles'],
where: [
{
libraryId: req.library.id
},
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = "${metadataFilename}")`), {
Sequelize.where(Sequelize.literal(metadataFilenameCountQuery), {
[Sequelize.Op.gte]: 1
})
]
],
replacements: {
metadataFilename
}
})
if (!libraryItemsWithMetadata.length) {
Logger.info(`[LibraryController] No ${metadataFilename} files found to remove`)

54
server/controllers/MeController.js
View file

@ -155,21 +155,23 @@ class MeController {
* @param {Response} res
*/
async getListeningSessions(req, res) {
const listeningSessions = await this.getUserListeningSessionsHelper(req.user.id)
const startedAt = Date.now()
const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10
const page = toNumber(req.query.page, 0)
const start = page * itemsPerPage
const sessions = listeningSessions.slice(start, start + itemsPerPage)
Logger.debug(
`[MeController] /api/me/listening-sessions user="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} start`
)
const payload = {
total: listeningSessions.length,
numPages: Math.ceil(listeningSessions.length / itemsPerPage),
const payload = await this.getUserListeningSessionsPageHelper(
req.user.id,
page,
itemsPerPage,
sessions
}
itemsPerPage
)
Logger.debug(
`[MeController] /api/me/listening-sessions user="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms`
)
res.json(payload)
}
@ -183,6 +185,7 @@ class MeController {
* @param {Response} res
*/
async getItemListeningSessions(req, res) {
const startedAt = Date.now()
const libraryItem = await Database.libraryItemModel.getExpandedById(req.params.libraryItemId)
const episode = await Database.podcastEpisodeModel.findByPk(req.params.episodeId)
@ -198,21 +201,23 @@ class MeController {
}
const mediaItemId = episode?.id || libraryItem.mediaId
let listeningSessions = await this.getUserItemListeningSessionsHelper(req.user.id, mediaItemId)
const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10
const page = toNumber(req.query.page, 0)
const start = page * itemsPerPage
const sessions = listeningSessions.slice(start, start + itemsPerPage)
Logger.debug(
`[MeController] /api/me/item/listening-sessions user="${req.user.id}" libraryItem="${req.params.libraryItemId}" episode="${req.params.episodeId || ''}" page=${page} itemsPerPage=${itemsPerPage} start`
)
const payload = {
total: listeningSessions.length,
numPages: Math.ceil(listeningSessions.length / itemsPerPage),
const payload = await this.getUserListeningSessionsPageHelper(
req.user.id,
page,
itemsPerPage,
sessions
}
mediaItemId
)
Logger.debug(
`[MeController] /api/me/item/listening-sessions user="${req.user.id}" libraryItem="${req.params.libraryItemId}" episode="${req.params.episodeId || ''}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms`
)
res.json(payload)
}
@ -226,7 +231,16 @@ class MeController {
* @param {Response} res
*/
async getListeningStats(req, res) {
const listeningStats = await this.getUserListeningStatsHelpers(req.user.id)
const startedAt = Date.now()
const minified = req.query.minified === '1' || req.query.minified === 'true'
Logger.debug(`[MeController] /api/me/listening-stats user="${req.user.id}" start`)
const listeningStats = await this.getUserListeningStatsHelpers(req.user.id, {
includeItems: !minified,
includeRecentSessions: !minified
})
Logger.debug(
`[MeController] /api/me/listening-stats user="${req.user.id}" minified=${minified} totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms`
)
res.json(listeningStats)
}

37
server/controllers/UserController.js
View file

@ -432,14 +432,20 @@ class UserController {
* @param {Response} res
*/
async getListeningSessions(req, res) {
var listeningSessions = await this.getUserListeningSessionsHelper(req.params.id)
const startedAt = Date.now()
const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10
const page = toNumber(req.query.page, 0)
Logger.debug(
`[UserController] /api/users/${req.params.id}/listening-sessions reqUser="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} start`
)
const payload = await this.getUserListeningSessionsPageHelper(
req.params.id,
page,
itemsPerPage
)
const start = page * itemsPerPage
// Map user to sessions to match the format of the sessions endpoint
const sessions = listeningSessions.slice(start, start + itemsPerPage).map((session) => {
payload.sessions = payload.sessions.map((session) => {
return {
...session,
user: {
@ -449,13 +455,9 @@ class UserController {
}
})
const payload = {
total: listeningSessions.length,
numPages: Math.ceil(listeningSessions.length / itemsPerPage),
page,
itemsPerPage,
sessions
}
Logger.debug(
`[UserController] /api/users/${req.params.id}/listening-sessions reqUser="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms`
)
res.json(payload)
}
@ -469,7 +471,18 @@ class UserController {
* @param {Response} res
*/
async getListeningStats(req, res) {
var listeningStats = await this.getUserListeningStatsHelpers(req.params.id)
const startedAt = Date.now()
const minified = req.query.minified === '1' || req.query.minified === 'true'
Logger.debug(
`[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" start`
)
var listeningStats = await this.getUserListeningStatsHelpers(req.params.id, {
includeItems: !minified,
includeRecentSessions: !minified
})
Logger.debug(
`[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" minified=${minified} totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms`
)
res.json(listeningStats)
}

326
server/managers/BackupManager.js
View file

@ -1,3 +1,4 @@
const childProcess = require('child_process')
const sqlite3 = require('sqlite3')
const Path = require('path')
const Logger = require('../Logger')
@ -47,6 +48,30 @@ class BackupManager {
return global.ServerSettings.maxBackupSize || Infinity
}
get databaseBackupConfig() {
if (Database.isPostgresDialect()) {
return {
dialect: 'postgres',
entryName: 'absdatabase.postgres.dump'
}
}
return {
dialect: 'sqlite',
entryName: 'absdatabase.sqlite'
}
}
getBackupDialect(backup) {
if (backup.key === 'postgres') return 'postgres'
if (backup.key === 'sqlite' || !backup.key) return 'sqlite'
return null
}
getBackupEntryName(dialect) {
return dialect === 'postgres' ? 'absdatabase.postgres.dump' : 'absdatabase.sqlite'
}
async init() {
try {
const backupsDirExists = await fs.pathExists(this.backupPath)
@ -130,13 +155,6 @@ class BackupManager {
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(400).send('Failed to read backup file - backup might not be a valid .zip file')
}
if (!entries['absdatabase.sqlite']) {
Logger.error(`[BackupManager] Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.`)
await zip.close().catch(() => {})
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(500).send('Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.')
}
const detailsEntry = entries['details']
if (!detailsEntry) {
Logger.error('[BackupManager] Invalid backup - missing details entry')
@ -151,10 +169,26 @@ class BackupManager {
return res.status(400).send('Invalid backup file - details entry too large')
}
const data = await zip.entryData('details')
const details = data.toString('utf8').split('\n')
let backup
try {
const data = await zip.entryData('details')
const details = data.toString('utf8').split('\n')
backup = new Backup({ details, fullPath: tempPath })
} catch (error) {
Logger.error(`[BackupManager] Invalid backup with no readable details file`, tempPath, error)
await zip.close().catch(() => {})
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(400).send('Invalid backup file. Missing readable details.')
}
const backup = new Backup({ details, fullPath: tempPath })
const backupDialect = this.getBackupDialect(backup)
const databaseEntryName = this.getBackupEntryName(backupDialect)
if (!backupDialect || !entries[databaseEntryName]) {
Logger.error(`[BackupManager] Invalid backup with no ${databaseEntryName} file - unsupported database backup.`)
await zip.close().catch(() => {})
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(500).send(`Invalid backup file. Does not include ${databaseEntryName}.`)
}
if (!backup.serverVersion) {
Logger.error(`[BackupManager] Invalid backup with no server version - might be a backup created before version 2.0.0`)
@ -204,8 +238,24 @@ class BackupManager {
const entries = await zip.entries()
const backupDialect = this.getBackupDialect(backup)
const currentDialect = Database.isPostgresDialect() ? 'postgres' : 'sqlite'
if (!backupDialect) {
await zip.close()
return res.status(500).send('Invalid backup file. Unsupported database backup format.')
}
if (backupDialect !== currentDialect) {
await zip.close()
return res.status(400).send(`Cannot apply a ${backupDialect} backup while using the ${currentDialect} database.`)
}
if (backupDialect === 'postgres') {
return this.requestApplyPostgresBackup(apiCacheManager, backup, zip, entries, res)
}
// Ensure backup has an absdatabase.sqlite file
if (!Object.keys(entries).includes('absdatabase.sqlite')) {
if (!Object.keys(entries).includes(this.getBackupEntryName(backupDialect))) {
Logger.error(`[BackupManager] Cannot apply old backup ${backup.fullPath}`)
await zip.close()
return res.status(500).send('Invalid backup file. Does not include absdatabase.sqlite. This might be from an older Audiobookshelf server.')
@ -266,6 +316,70 @@ class BackupManager {
SocketAuthority.emitter('backup_applied')
}
async requestApplyPostgresBackup(apiCacheManager, backup, zip, entries, res) {
const databaseEntryName = this.getBackupEntryName('postgres')
if (!Object.keys(entries).includes(databaseEntryName)) {
Logger.error(`[BackupManager] Cannot apply Postgres backup ${backup.fullPath}`)
await zip.close()
return res.status(500).send(`Invalid backup file. Does not include ${databaseEntryName}.`)
}
const tempDumpPath = Path.join(global.ConfigPath, 'absdatabase-postgres-temp.dump')
let reconnected = false
let zipClosed = false
const closeZip = async () => {
if (zipClosed) return
zipClosed = true
await zip.close()
}
try {
await fs.remove(tempDumpPath)
await zip.extract(databaseEntryName, tempDumpPath)
if (!(await fs.pathExists(tempDumpPath))) {
await closeZip()
return res.status(500).send('Failed to extract Postgres database dump from backup')
}
await Database.disconnect()
await this.restorePostgresDb(tempDumpPath)
await fs.ensureDir(this.ItemsMetadataPath)
await zip.extract('metadata-items/', this.ItemsMetadataPath)
await fs.ensureDir(this.AuthorsMetadataPath)
await zip.extract('metadata-authors/', this.AuthorsMetadataPath)
await closeZip()
await Database.reconnect()
reconnected = true
await apiCacheManager.reset()
await CacheManager.purgeAll()
res.sendStatus(200)
SocketAuthority.emitter('backup_applied')
} catch (error) {
Logger.error(`[BackupManager] Failed to apply Postgres backup`, error)
try {
await closeZip()
} catch (closeError) {
Logger.error(`[BackupManager] Failed to close Postgres backup archive`, closeError)
}
if (!reconnected) {
try {
await Database.reconnect()
} catch (reconnectError) {
Logger.error(`[BackupManager] Failed to reconnect after Postgres backup apply`, reconnectError)
}
}
return res.status(500).send(`Failed to apply Postgres backup: ${error?.message || 'Unknown Error'}`)
} finally {
await fs.remove(tempDumpPath)
}
}
async loadBackups() {
try {
const filesInDir = await fs.readdir(this.backupPath)
@ -276,7 +390,7 @@ class BackupManager {
const fullFilePath = Path.join(this.backupPath, filename)
let zip = null
let data = null
let backup = null
try {
zip = new StreamZip.async({ file: fullFilePath })
const entries = await zip.entries()
@ -293,17 +407,24 @@ class BackupManager {
continue
}
data = await zip.entryData('details')
const data = await zip.entryData('details')
const details = data.toString('utf8').split('\n')
backup = new Backup({ details, fullPath: fullFilePath })
const backupDialect = this.getBackupDialect(backup)
const databaseEntryName = this.getBackupEntryName(backupDialect)
if (!backupDialect || !entries[databaseEntryName]) {
Logger.error(`[BackupManager] Unsupported database backup format found "${backup.filename}"`)
await zip.close().catch(() => {})
continue
}
} catch (error) {
Logger.error(`[BackupManager] Failed to unzip backup "${fullFilePath}"`, error)
if (zip) await zip.close().catch(() => {})
continue
}
const details = data.toString('utf8').split('\n')
const backup = new Backup({ details, fullPath: fullFilePath })
if (!backup.serverVersion) {
// Backups before v2
Logger.error(`[BackupManager] Old unsupported backup was found "${backup.filename}"`)
@ -333,33 +454,34 @@ class BackupManager {
async runBackup() {
// Check if Metadata Path is inside Config Path (otherwise there will be an infinite loop as the archiver tries to zip itself)
Logger.info(`[BackupManager] Running Backup`)
const databaseBackupConfig = this.databaseBackupConfig
const newBackup = new Backup()
newBackup.setData(this.backupPath)
newBackup.setData(this.backupPath, databaseBackupConfig.dialect)
await fs.ensureDir(this.AuthorsMetadataPath)
// Create backup sqlite file
const sqliteBackupPath = await this.backupSqliteDb(newBackup).catch((error) => {
Logger.error(`[BackupManager] Failed to backup sqlite db`, error)
// Create a database dump
const databaseBackupPath = await this.backupDatabase(newBackup).catch((error) => {
Logger.error(`[BackupManager] Failed to backup ${databaseBackupConfig.dialect} database`, error)
const errorMsg = error?.message || error || 'Unknown Error'
NotificationManager.onBackupFailed(errorMsg)
return false
})
if (!sqliteBackupPath) {
if (!databaseBackupPath) {
return false
}
// Zip sqlite file, /metadata/items, and /metadata/authors folders
const zipResult = await this.zipBackup(sqliteBackupPath, newBackup).catch((error) => {
// Zip database dump, /metadata/items, and /metadata/authors folders
const zipResult = await this.zipBackup(databaseBackupPath, newBackup, databaseBackupConfig.entryName).catch((error) => {
Logger.error(`[BackupManager] Backup Failed ${error}`)
const errorMsg = error?.message || error || 'Unknown Error'
NotificationManager.onBackupFailed(errorMsg)
return false
})
// Remove sqlite backup
await fs.remove(sqliteBackupPath)
// Remove temporary database dump
await fs.remove(databaseBackupPath)
if (!zipResult) return false
@ -390,6 +512,10 @@ class BackupManager {
return true
}
backupDatabase(backup) {
return this.databaseBackupConfig.dialect === 'postgres' ? this.backupPostgresDb(backup) : this.backupSqliteDb(backup)
}
async removeBackup(backup) {
try {
Logger.debug(`[BackupManager] Removing Backup "${backup.fullPath}"`)
@ -406,29 +532,147 @@ class BackupManager {
* @param {Backup} backup
*/
backupSqliteDb(backup) {
const db = new sqlite3.Database(Database.dbPath)
const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.sqlite`)
return new Promise(async (resolve, reject) => {
const backup = db.backup(dbFilePath)
backup.step(-1)
backup.finish()
let db
let sqliteBackup
let settled = false
// Max time ~2 mins
for (let i = 0; i < 240; i++) {
if (backup.completed) {
return resolve(dbFilePath)
} else if (backup.failed) {
return reject(backup.message || 'Unknown failure reason')
const finish = (error, result) => {
if (settled) return
settled = true
if (db) {
db.close(() => {
if (error) reject(error)
else resolve(result)
})
} else if (error) {
reject(error)
} else {
resolve(result)
}
await new Promise((r) => setTimeout(r, 500))
}
Logger.error(`[BackupManager] Backup sqlite timed out`)
reject('Backup timed out')
const pollBackup = async () => {
// Max time ~2 mins
for (let i = 0; i < 240; i++) {
if (sqliteBackup.completed) {
return finish(null, dbFilePath)
} else if (sqliteBackup.failed) {
return finish(sqliteBackup.message || 'Unknown failure reason')
}
await new Promise((r) => setTimeout(r, 500))
}
Logger.error(`[BackupManager] Backup sqlite timed out`)
finish('Backup timed out')
}
const startBackup = () => {
try {
sqliteBackup = db.backup(dbFilePath)
sqliteBackup.step(-1)
sqliteBackup.finish()
pollBackup().catch(finish)
} catch (error) {
finish(error)
}
}
db = new sqlite3.Database(Database.dbPath, (error) => {
if (error) return finish(error)
startBackup()
})
db.on('error', finish)
})
}
zipBackup(sqliteBackupPath, backup) {
/**
* Build pg_dump/pg_restore connection arguments from DATABASE_URL without
* exposing credentials in argv. execFile error messages and the host process
* list include argv, so the password is passed via PGPASSWORD env instead.
*/
getPostgresConnection() {
let dbUrl
try {
dbUrl = new URL(Database.dbPath)
} catch (error) {
throw new Error('DATABASE_URL must be a valid postgres connection URI to run backups')
}
const args = ['--host', dbUrl.hostname, '--dbname', decodeURIComponent(dbUrl.pathname.replace(/^\//, ''))]
if (dbUrl.port) args.push('--port', dbUrl.port)
if (dbUrl.username) args.push('--username', decodeURIComponent(dbUrl.username))
// Redact both the percent-encoded and decoded password from any error output
const decodedPassword = dbUrl.password ? decodeURIComponent(dbUrl.password) : null
const secrets = dbUrl.password ? [dbUrl.password, decodedPassword] : []
const env = decodedPassword ? { ...process.env, PGPASSWORD: decodedPassword } : process.env
return { args, env, secrets }
}
backupPostgresDb(backup) {
const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.postgres.dump`)
return this.runPostgresCommand('pg_dump', [
'--format=custom',
'--no-owner',
'--no-acl',
'--file',
dbFilePath
])
.then(() => dbFilePath)
.catch(async (error) => {
await fs.remove(dbFilePath)
throw error
})
}
restorePostgresDb(dbFilePath) {
return this.runPostgresCommand('pg_restore', [
'--clean',
'--if-exists',
'--exit-on-error',
'--single-transaction',
'--no-owner',
'--no-acl',
dbFilePath
])
}
runPostgresCommand(command, args) {
return new Promise((resolve, reject) => {
let connection
try {
connection = this.getPostgresConnection()
} catch (error) {
return reject(error)
}
const redact = (text) => {
if (typeof text !== 'string') return text
return connection.secrets.reduce((redacted, secret) => redacted.split(secret).join('***'), text)
}
const options = {
maxBuffer: 10 * 1024 * 1024,
// Kill after 30 mins (e.g. lock waits) - a killed restore rolls back via --single-transaction
timeout: 30 * 60 * 1000,
env: connection.env
}
childProcess.execFile(command, [...args, ...connection.args], options, (error, stdout, stderr) => {
if (error) {
error.message = redact(error.message)
if (error.cmd) error.cmd = redact(error.cmd)
error.stderr = redact(stderr)
return reject(error)
}
resolve({ stdout, stderr })
})
})
}
zipBackup(databaseBackupPath, backup, databaseEntryName = 'absdatabase.sqlite') {
return new Promise((resolve, reject) => {
// create a file to stream archive data to
const output = fs.createWriteStream(backup.fullPath)
@ -492,7 +736,7 @@ class BackupManager {
// pipe archive data to the file
archive.pipe(output)
archive.file(sqliteBackupPath, { name: 'absdatabase.sqlite' })
archive.file(databaseBackupPath, { name: databaseEntryName })
archive.directory(this.ItemsMetadataPath, 'metadata-items')
archive.directory(this.AuthorsMetadataPath, 'metadata-authors')

72
server/managers/MigrationManager.js
View file

@ -94,20 +94,26 @@ class MigrationManager {
// Only proceed with migration if there are migrations to run
if (migrationsToRun.length > 0) {
const dialect = typeof this.sequelize.getDialect === 'function' ? this.sequelize.getDialect() : 'sqlite'
const isSqlite = !dialect || dialect === 'sqlite'
const originalDbPath = path.join(this.configPath, 'absdatabase.sqlite')
const backupDbPath = path.join(this.configPath, 'absdatabase.backup.sqlite')
try {
Logger.info(`[MigrationManager] Migrating database ${migrationDirection} to version ${this.serverVersion}`)
Logger.info(`[MigrationManager] Migrations to run: ${migrationsToRun.join(', ')}`)
// Create a backup copy of the SQLite database before starting migrations
await fs.copy(originalDbPath, backupDbPath)
Logger.info('Created a backup of the original database.')
if (isSqlite) {
// Create a backup copy of the SQLite database before starting migrations
await fs.copy(originalDbPath, backupDbPath)
Logger.info('Created a backup of the original database.')
}
// Run migrations
await this.umzug[migrationDirection]({ migrations: migrationsToRun, rerun: 'ALLOW' })
// Clean up the backup
await fs.remove(backupDbPath)
if (isSqlite) {
// Clean up the backup
await fs.remove(backupDbPath)
}
Logger.info('[MigrationManager] Migrations successfully applied to the original database.')
} catch (error) {
@ -115,13 +121,15 @@ class MigrationManager {
await this.sequelize.close()
// Step 3: If migration fails, save the failed original and restore the backup
const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite')
await fs.move(originalDbPath, failedDbPath, { overwrite: true })
Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.')
if (isSqlite) {
// Step 3: If migration fails, save the failed original and restore the backup
const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite')
await fs.move(originalDbPath, failedDbPath, { overwrite: true })
Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.')
await fs.move(backupDbPath, originalDbPath, { overwrite: true })
Logger.info('[MigrationManager] Restored the original database from the backup.')
await fs.move(backupDbPath, originalDbPath, { overwrite: true })
Logger.info('[MigrationManager] Restored the original database from the backup.')
}
Logger.info('[MigrationManager] Migration failed. Exiting Audiobookshelf with code 1.')
process.exit(1)
@ -191,29 +199,27 @@ class MigrationManager {
}
async fetchVersionsFromDatabase() {
const migrationsMetaTable = MigrationManager.MIGRATIONS_META_TABLE
await this.checkOrCreateMigrationsMetaTable()
const [{ version }] = await this.sequelize.query("SELECT value as version FROM :migrationsMeta WHERE key = 'version'", {
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
const [versionRow] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, {
type: Sequelize.QueryTypes.SELECT
})
this.databaseVersion = version
this.databaseVersion = versionRow?.version
const [{ maxVersion }] = await this.sequelize.query("SELECT value as maxVersion FROM :migrationsMeta WHERE key = 'maxVersion'", {
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
const [maxVersionRow] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, {
type: Sequelize.QueryTypes.SELECT
})
this.maxVersion = maxVersion
this.maxVersion = maxVersionRow?.maxVersion || maxVersionRow?.maxversion
}
async checkOrCreateMigrationsMetaTable() {
const queryInterface = this.sequelize.getQueryInterface()
let migrationsMetaTableExists = await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE)
let migrationsMetaTableExists = await this.tableExists(MigrationManager.MIGRATIONS_META_TABLE)
// If the table exists, check that the `version` and `maxVersion` rows exist
if (migrationsMetaTableExists) {
const [{ count }] = await this.sequelize.query("SELECT COUNT(*) as count FROM :migrationsMeta WHERE key IN ('version', 'maxVersion')", {
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM ${MigrationManager.MIGRATIONS_META_TABLE} WHERE key IN ('version', 'maxVersion')`, {
type: Sequelize.QueryTypes.SELECT
})
if (count < 2) {
@ -241,14 +247,28 @@ class MigrationManager {
allowNull: false
}
})
await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", {
replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
await this.sequelize.query(`INSERT INTO ${MigrationManager.MIGRATIONS_META_TABLE} (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, {
replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0' },
type: Sequelize.QueryTypes.INSERT
})
Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`)
}
}
async tableExists(tableName) {
const queryInterface = this.sequelize.getQueryInterface()
if (typeof queryInterface.tableExists === 'function') {
return queryInterface.tableExists(tableName)
}
const tables = await queryInterface.showAllTables()
return tables.some((table) => {
if (typeof table === 'string') return table === tableName
if (table?.tableName) return table.tableName === tableName
return false
})
}
extractVersionFromTag(tag) {
if (!tag) return null
const versionMatch = tag.match(/^v?(\d+\.\d+\.\d+)/)
@ -299,8 +319,8 @@ class MigrationManager {
async updateMaxVersion() {
try {
await this.sequelize.query("UPDATE :migrationsMeta SET value = :maxVersion WHERE key = 'maxVersion'", {
replacements: { maxVersion: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :maxVersion WHERE key = 'maxVersion'`, {
replacements: { maxVersion: this.serverVersion },
type: Sequelize.QueryTypes.UPDATE
})
} catch (error) {
@ -311,8 +331,8 @@ class MigrationManager {
async updateDatabaseVersion() {
try {
await this.sequelize.query("UPDATE :migrationsMeta SET value = :version WHERE key = 'version'", {
replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
await this.sequelize.query(`UPDATE ${MigrationManager.MIGRATIONS_META_TABLE} SET value = :version WHERE key = 'version'`, {
replacements: { version: this.serverVersion },
type: Sequelize.QueryTypes.UPDATE
})
} catch (error) {

34
server/migrations/v2.15.0-series-column-unique.js
View file

@ -19,12 +19,21 @@ async function up({ context: { queryInterface, logger } }) {
logger.info('[2.15.0 migration] UPGRADE BEGIN: 2.15.0-series-column-unique ')
// Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0
logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues')
await queryInterface.sequelize.query('REINDEX NOCASE;')
if (queryInterface.sequelize.getDialect() === 'sqlite') {
logger.info('[2.15.0 migration] Reindexing NOCASE indices to fix potential hidden corruption issues')
await queryInterface.sequelize.query('REINDEX NOCASE;')
} else {
logger.info('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect')
}
// Check if the unique index already exists
const seriesIndexes = await queryInterface.showIndex('Series')
if (seriesIndexes.some((index) => index.name === 'unique_series_name_per_library')) {
if (
seriesIndexes.some((index) => {
const indexName = index?.name || index?.indexName || ''
return String(indexName).toLowerCase() === 'unique_series_name_per_library'
})
) {
logger.info('[2.15.0 migration] Unique index on Series.name and Series.libraryId already exists')
logger.info('[2.15.0 migration] UPGRADE END: 2.15.0-series-column-unique ')
return
@ -181,11 +190,20 @@ async function up({ context: { queryInterface, logger } }) {
logger.info(`[2.15.0 migration] Deduplication complete`)
// Create a unique index based on the name and library ID for the `Series` table
await queryInterface.addIndex('Series', ['name', 'libraryId'], {
unique: true,
name: 'unique_series_name_per_library'
})
logger.info('[2.15.0 migration] Added unique index on Series.name and Series.libraryId')
try {
await queryInterface.addIndex('Series', ['name', 'libraryId'], {
unique: true,
name: 'unique_series_name_per_library'
})
logger.info('[2.15.0 migration] Added unique index on Series.name and Series.libraryId')
} catch (error) {
const alreadyExists =
(error?.name === 'SequelizeDatabaseError' && /already exists/i.test(error?.message || '')) ||
error?.original?.code === '42P07'
if (!alreadyExists) throw error
logger.info('[2.15.0 migration] Unique index on Series.name and Series.libraryId already exists')
}
logger.info('[2.15.0 migration] UPGRADE END: 2.15.0-series-column-unique ')
}

6
server/migrations/v2.15.1-reindex-nocase.js
View file

@ -17,6 +17,12 @@ async function up({ context: { queryInterface, logger } }) {
// Upwards migration script
logger.info('[2.15.1 migration] UPGRADE BEGIN: 2.15.1-reindex-nocase ')
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect')
logger.info('[2.15.1 migration] UPGRADE END: 2.15.1-reindex-nocase ')
return
}
// Run reindex nocase to fix potential corruption issues due to the bad sqlite extension introduced in v2.12.0
logger.info('[2.15.1 migration] Reindexing NOCASE indices to fix potential hidden corruption issues')
await queryInterface.sequelize.query('REINDEX NOCASE;')

52
server/migrations/v2.15.2-index-creation.js
View file

@ -20,10 +20,8 @@ async function up({ context: { queryInterface, logger } }) {
// Create index for bookAuthors
logger.info('[2.15.2 migration] Creating index for bookAuthors')
const bookAuthorsIndexes = await queryInterface.showIndex('bookAuthors')
if (!bookAuthorsIndexes.some((index) => index.name === 'bookAuthor_authorId')) {
await queryInterface.addIndex('bookAuthors', ['authorId'], {
name: 'bookAuthor_authorId'
})
if (!hasIndex(bookAuthorsIndexes, 'bookAuthor_authorId')) {
await addIndexIfMissing(queryInterface, 'bookAuthors', ['authorId'], 'bookAuthor_authorId', logger)
} else {
logger.info('[2.15.2 migration] Index bookAuthor_authorId already exists')
}
@ -31,25 +29,21 @@ async function up({ context: { queryInterface, logger } }) {
// Create index for bookSeries
logger.info('[2.15.2 migration] Creating index for bookSeries')
const bookSeriesIndexes = await queryInterface.showIndex('bookSeries')
if (!bookSeriesIndexes.some((index) => index.name === 'bookSeries_seriesId')) {
await queryInterface.addIndex('bookSeries', ['seriesId'], {
name: 'bookSeries_seriesId'
})
if (!hasIndex(bookSeriesIndexes, 'bookSeries_seriesId')) {
await addIndexIfMissing(queryInterface, 'bookSeries', ['seriesId'], 'bookSeries_seriesId', logger)
} else {
logger.info('[2.15.2 migration] Index bookSeries_seriesId already exists')
}
// Delete existing podcastEpisode index
logger.info('[2.15.2 migration] Deleting existing podcastEpisode index')
await queryInterface.removeIndex('podcastEpisodes', 'podcast_episodes_created_at')
await removeIndexIfExists(queryInterface, 'podcastEpisodes', 'podcast_episodes_created_at', logger)
// Create index for podcastEpisode and createdAt
logger.info('[2.15.2 migration] Creating index for podcastEpisode and createdAt')
const podcastEpisodesIndexes = await queryInterface.showIndex('podcastEpisodes')
if (!podcastEpisodesIndexes.some((index) => index.name === 'podcastEpisode_createdAt_podcastId')) {
await queryInterface.addIndex('podcastEpisodes', ['createdAt', 'podcastId'], {
name: 'podcastEpisode_createdAt_podcastId'
})
if (!hasIndex(podcastEpisodesIndexes, 'podcastEpisode_createdAt_podcastId')) {
await addIndexIfMissing(queryInterface, 'podcastEpisodes', ['createdAt', 'podcastId'], 'podcastEpisode_createdAt_podcastId', logger)
} else {
logger.info('[2.15.2 migration] Index podcastEpisode_createdAt_podcastId already exists')
}
@ -78,7 +72,7 @@ async function down({ context: { queryInterface, logger } }) {
// Delete existing podcastEpisode index
logger.info('[2.15.2 migration] Deleting existing podcastEpisode index')
await queryInterface.removeIndex('podcastEpisodes', 'podcastEpisode_createdAt_podcastId')
await removeIndexIfExists(queryInterface, 'podcastEpisodes', 'podcastEpisode_createdAt_podcastId', logger)
// Create index for podcastEpisode and createdAt
logger.info('[2.15.2 migration] Creating original index for podcastEpisode createdAt')
@ -91,3 +85,33 @@ async function down({ context: { queryInterface, logger } }) {
}
module.exports = { up, down }
async function removeIndexIfExists(queryInterface, tableName, indexName, logger) {
const indexes = await queryInterface.showIndex(tableName)
const hasIndexWithName = hasIndex(indexes, indexName)
if (!hasIndexWithName) {
logger.info(`[2.15.2 migration] Index ${indexName} does not exist, skipping removeIndex`)
return
}
await queryInterface.removeIndex(tableName, indexName)
}
function hasIndex(indexes, indexName) {
const expected = String(indexName || '').toLowerCase()
return indexes.some((index) => String(index?.name || index?.indexName || '').toLowerCase() === expected)
}
async function addIndexIfMissing(queryInterface, tableName, fields, indexName, logger) {
try {
await queryInterface.addIndex(tableName, fields, { name: indexName })
} catch (error) {
const alreadyExists =
(error?.name === 'SequelizeDatabaseError' && /already exists/i.test(error?.message || '')) ||
error?.original?.code === '42P07'
if (!alreadyExists) throw error
logger.info(`[2.15.2 migration] Index ${indexName} already exists`)
}
}

6
server/migrations/v2.17.3-fk-constraints.js
View file

@ -18,6 +18,12 @@ async function up({ context: { queryInterface, logger } }) {
// Upwards migration script
logger.info('[2.17.3 migration] UPGRADE BEGIN: 2.17.3-fk-constraints')
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect')
logger.info('[2.17.3 migration] UPGRADE END: 2.17.3-fk-constraints')
return
}
const execQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize)
// Disable foreign key constraints for the next sequence of operations

17
server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.js
View file

@ -56,15 +56,23 @@ async function down({ context: { queryInterface, logger } }) {
}
async function getServerSettings(queryInterface, logger) {
const result = await queryInterface.sequelize.query('SELECT value FROM settings WHERE key = "server-settings";')
const result = await queryInterface.sequelize.query('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
if (!result[0].length) {
logger.error('[2.17.4 migration] Server settings not found')
throw new Error('Server settings not found')
}
const settingsValue = result[0][0].value
if (settingsValue && typeof settingsValue === 'object') {
return settingsValue
}
let serverSettings = null
try {
serverSettings = JSON.parse(result[0][0].value)
serverSettings = JSON.parse(settingsValue)
} catch (error) {
logger.error('[2.17.4 migration] Error parsing server settings:', error)
throw error
@ -74,9 +82,10 @@ async function getServerSettings(queryInterface, logger) {
}
async function updateServerSettings(queryInterface, logger, serverSettings) {
await queryInterface.sequelize.query('UPDATE settings SET value = :value WHERE key = "server-settings";', {
await queryInterface.sequelize.query('UPDATE settings SET value = :value WHERE key = :settingsKey;', {
replacements: {
value: JSON.stringify(serverSettings)
value: JSON.stringify(serverSettings),
settingsKey: 'server-settings'
}
})
}

12
server/migrations/v2.19.1-copy-title-to-library-items.js
View file

@ -25,6 +25,12 @@ async function up({ context: { queryInterface, logger } }) {
// Upwards migration script
logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`)
logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
return
}
await addColumn(queryInterface, logger, 'libraryItems', 'title', { type: queryInterface.sequelize.Sequelize.STRING, allowNull: true })
await copyColumn(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId')
await addTrigger(queryInterface, logger, 'books', 'title', 'id', 'libraryItems', 'title', 'mediaId')
@ -51,6 +57,12 @@ async function down({ context: { queryInterface, logger } }) {
// Downward migration script
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`)
logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
return
}
await removeIndex(queryInterface, logger, 'libraryItems', ['libraryId', 'mediaType', 'title'])
await removeTrigger(queryInterface, logger, 'libraryItems', 'title')
await removeColumn(queryInterface, logger, 'libraryItems', 'title')

12
server/migrations/v2.19.4-improve-podcast-queries.js
View file

@ -26,6 +26,12 @@ async function up({ context: { queryInterface, logger } }) {
// Upwards migration script
logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`)
logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
return
}
// Add numEpisodes column to podcasts table
await addColumn(queryInterface, logger, 'podcasts', 'numEpisodes', { type: queryInterface.sequelize.Sequelize.INTEGER, allowNull: false, defaultValue: 0 })
@ -60,6 +66,12 @@ async function down({ context: { queryInterface, logger } }) {
// Downward migration script
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`)
logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
return
}
// Remove triggers from libraryItems
await removeTrigger(queryInterface, logger, 'podcasts', 'title', 'libraryItems', 'title')
await removeTrigger(queryInterface, logger, 'podcasts', 'titleIgnorePrefix', 'libraryItems', 'titleIgnorePrefix')

10
server/migrations/v2.20.0-improve-author-sort-queries.js
View file

@ -39,6 +39,11 @@ const authorsJoin = `${authors} JOIN ${bookAuthors} ON ${authors}.id = ${bookAut
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
*/
async function up({ context: { queryInterface, logger } }) {
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`)
return
}
const helper = new MigrationHelper(queryInterface, logger)
// Upwards migration script
@ -72,6 +77,11 @@ async function up({ context: { queryInterface, logger } }) {
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
*/
async function down({ context: { queryInterface, logger } }) {
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`)
return
}
// Downward migration script
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)

12
server/migrations/v2.35.0-add-last-refresh-token.js
View file

@ -22,8 +22,10 @@ async function up({ context: { queryInterface, logger } }) {
if (await queryInterface.tableExists('sessions')) {
const tableDescription = await queryInterface.describeTable('sessions')
// Postgres folds unquoted identifiers to lowercase, so match column names case-insensitively
const hasColumn = (name) => Object.keys(tableDescription).some((column) => column.toLowerCase() === name.toLowerCase())
if (!tableDescription.lastRefreshToken) {
if (!hasColumn('lastRefreshToken')) {
logger.info(`${loggerPrefix} Adding lastRefreshToken column to sessions table`)
await queryInterface.addColumn('sessions', 'lastRefreshToken', {
type: queryInterface.sequelize.Sequelize.DataTypes.STRING,
@ -33,7 +35,7 @@ async function up({ context: { queryInterface, logger } }) {
logger.info(`${loggerPrefix} lastRefreshToken column already exists in sessions table`)
}
if (!tableDescription.lastRefreshTokenExpiresAt) {
if (!hasColumn('lastRefreshTokenExpiresAt')) {
logger.info(`${loggerPrefix} Adding lastRefreshTokenExpiresAt column to sessions table`)
await queryInterface.addColumn('sessions', 'lastRefreshTokenExpiresAt', {
type: queryInterface.sequelize.Sequelize.DataTypes.DATE,
@ -60,15 +62,17 @@ async function down({ context: { queryInterface, logger } }) {
if (await queryInterface.tableExists('sessions')) {
const tableDescription = await queryInterface.describeTable('sessions')
// Postgres folds unquoted identifiers to lowercase, so match column names case-insensitively
const hasColumn = (name) => Object.keys(tableDescription).some((column) => column.toLowerCase() === name.toLowerCase())
if (tableDescription.lastRefreshToken) {
if (hasColumn('lastRefreshToken')) {
logger.info(`${loggerPrefix} Removing lastRefreshToken column from sessions table`)
await queryInterface.removeColumn('sessions', 'lastRefreshToken')
} else {
logger.info(`${loggerPrefix} lastRefreshToken column does not exist in sessions table`)
}
if (tableDescription.lastRefreshTokenExpiresAt) {
if (hasColumn('lastRefreshTokenExpiresAt')) {
logger.info(`${loggerPrefix} Removing lastRefreshTokenExpiresAt column from sessions table`)
await queryInterface.removeColumn('sessions', 'lastRefreshTokenExpiresAt')
} else {

9
server/models/Author.js
View file

@ -129,6 +129,8 @@ class Author extends Model {
* @param {import('../Database').sequelize} sequelize
*/
static init(sequelize) {
const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' }
super.init(
{
id: {
@ -147,12 +149,7 @@ class Author extends Model {
modelName: 'author',
indexes: [
{
fields: [
{
name: 'name',
collate: 'NOCASE'
}
]
fields: [nameIndexField]
},
// {
// fields: [{

11
server/models/Book.js
View file

@ -138,6 +138,8 @@ class Book extends Model {
* @param {import('../Database').sequelize} sequelize
*/
static init(sequelize) {
const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' }
super.init(
{
id: {
@ -147,7 +149,7 @@ class Book extends Model {
},
title: DataTypes.STRING,
titleIgnorePrefix: DataTypes.STRING,
subtitle: DataTypes.STRING,
subtitle: DataTypes.TEXT,
publishedYear: DataTypes.STRING,
publishedDate: DataTypes.STRING,
publisher: DataTypes.STRING,
@ -172,12 +174,7 @@ class Book extends Model {
modelName: 'book',
indexes: [
{
fields: [
{
name: 'title',
collate: 'NOCASE'
}
]
fields: [titleIndexField]
},
// {
// fields: [{

13
server/models/LibraryItem.js
View file

@ -720,6 +720,11 @@ class LibraryItem extends Model {
* @param {import('../Database').sequelize} sequelize
*/
static init(sequelize) {
const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' }
const titleIgnorePrefixIndexField = sequelize.getDialect() === 'postgres' ? 'titleIgnorePrefix' : { name: 'titleIgnorePrefix', collate: 'NOCASE' }
const authorNamesFirstLastIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesFirstLast' : { name: 'authorNamesFirstLast', collate: 'NOCASE' }
const authorNamesLastFirstIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesLastFirst' : { name: 'authorNamesLastFirst', collate: 'NOCASE' }
super.init(
{
id: {
@ -768,16 +773,16 @@ class LibraryItem extends Model {
fields: ['libraryId', 'mediaType', 'createdAt']
},
{
fields: ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }]
fields: ['libraryId', 'mediaType', titleIndexField]
},
{
fields: ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }]
fields: ['libraryId', 'mediaType', titleIgnorePrefixIndexField]
},
{
fields: ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }]
fields: ['libraryId', 'mediaType', authorNamesFirstLastIndexField]
},
{
fields: ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }]
fields: ['libraryId', 'mediaType', authorNamesLastFirstIndexField]
},
{
fields: ['libraryId', 'mediaId', 'mediaType']

8
server/models/MediaProgress.js
View file

@ -257,7 +257,13 @@ class MediaProgress extends Model {
const escapedDate = this.sequelize.escape(new Date(progressPayload.lastUpdate))
Logger.info(`[MediaProgress] Manually setting updatedAt to ${escapedDate} (media item ${this.mediaItemId})`)
await this.sequelize.query(`UPDATE "mediaProgresses" SET "updatedAt" = ${escapedDate} WHERE "id" = '${this.id}'`)
await this.constructor.update(
{ updatedAt: new Date(progressPayload.lastUpdate) },
{
where: { id: this.id },
silent: true
}
)
await this.reload()
}

60
server/models/PlaybackSession.js
View file

@ -52,18 +52,54 @@ class PlaybackSession extends Model {
this.createdAt
}
static async getOldPlaybackSessions(where = null) {
static async getOldPlaybackSessions(where = null, options = {}) {
const {
limit = null,
offset = null,
order = [['updatedAt', 'DESC']],
includeDevice = true
} = options
const playbackSessions = await this.findAll({
where,
include: [
{
model: this.sequelize.models.device
}
]
limit,
offset,
order,
include: includeDevice
? [
{
model: this.sequelize.models.device
}
]
: undefined
})
return playbackSessions.map((session) => this.getOldPlaybackSession(session))
}
static countWithWhere(where = null) {
return this.count({ where })
}
static getPlaybackSessionsForStats(where = null) {
return this.findAll({
where,
order: [['updatedAt', 'DESC']],
attributes: [
'mediaItemId',
'mediaItemType',
'displayTitle',
'displayAuthor',
'timeListening',
'mediaMetadata',
'date',
'dayOfWeek',
'updatedAt',
'extraData'
],
raw: true
})
}
static async getById(sessionId) {
const playbackSession = await this.findByPk(sessionId, {
include: [
@ -190,7 +226,7 @@ class PlaybackSession extends Model {
currentTime: DataTypes.FLOAT,
serverVersion: DataTypes.STRING,
coverPath: DataTypes.STRING,
timeListening: DataTypes.INTEGER,
timeListening: DataTypes.FLOAT,
mediaMetadata: DataTypes.JSON,
date: DataTypes.STRING,
dayOfWeek: DataTypes.STRING,
@ -237,18 +273,20 @@ class PlaybackSession extends Model {
if (!Array.isArray(findResult)) findResult = [findResult]
for (const instance of findResult) {
const values = instance?.dataValues
if (instance.mediaItemType === 'book' && instance.book !== undefined) {
instance.mediaItem = instance.book
instance.dataValues.mediaItem = instance.dataValues.book
if (values) values.mediaItem = values.book
} else if (instance.mediaItemType === 'podcastEpisode' && instance.podcastEpisode !== undefined) {
instance.mediaItem = instance.podcastEpisode
instance.dataValues.mediaItem = instance.dataValues.podcastEpisode
if (values) values.mediaItem = values.podcastEpisode
}
// To prevent mistakes:
delete instance.book
delete instance.dataValues.book
if (values) delete values.book
delete instance.podcastEpisode
delete instance.dataValues.podcastEpisode
if (values) delete values.podcastEpisode
}
})
}

12
server/models/Series.js
View file

@ -1,6 +1,7 @@
const { DataTypes, Model, where, fn, col, literal } = require('sequelize')
const { getTitlePrefixAtEnd, getTitleIgnorePrefix } = require('../utils/index')
const { safeTextToDoubleExpression } = require('../utils/sqlDialectHelpers')
class Series extends Model {
constructor(values, options) {
@ -87,6 +88,8 @@ class Series extends Model {
* @param {import('../Database').sequelize} sequelize
*/
static init(sequelize) {
const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' }
super.init(
{
id: {
@ -103,12 +106,7 @@ class Series extends Model {
modelName: 'series',
indexes: [
{
fields: [
{
name: 'name',
collate: 'NOCASE'
}
]
fields: [nameIndexField]
},
// {
// fields: [{
@ -161,7 +159,7 @@ class Series extends Model {
}
}
],
order: [[literal('CAST(`bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]]
order: [[literal(`${safeTextToDoubleExpression(this.sequelize.getDialect() === 'postgres' ? '"bookSeries"."sequence"' : '`bookSeries.sequence`', this.sequelize)} ASC NULLS LAST`)]]
})
}

40
server/models/User.js
View file

@ -18,12 +18,16 @@ class UserCache {
}
getByEmail(email) {
const user = this.cache.find((u) => u.email === email)
if (!email) return null
const normalizedEmail = email.toLowerCase()
const user = this.cache.find((u) => u.email && u.email.toLowerCase() === normalizedEmail)
return user
}
getByUsername(username) {
const user = this.cache.find((u) => u.username === username)
if (!username) return null
const normalizedUsername = username.toLowerCase()
const user = this.cache.find((u) => u.username && u.username.toLowerCase() === normalizedUsername)
return user
}
@ -348,11 +352,13 @@ class User extends Model {
static async getUserByUsername(username) {
if (!username) return null
const cachedUser = userCache.getByUsername(username)
const normalizedUsername = username.toLowerCase()
const cachedUser = userCache.getByUsername(normalizedUsername)
if (cachedUser) return cachedUser
const user = await this.findOne({
where: sequelize.where(sequelize.fn('lower', sequelize.col('username')), username.toLowerCase()),
where: sequelize.where(sequelize.fn('LOWER', sequelize.col('username')), normalizedUsername),
include: this.sequelize.models.mediaProgress
})
@ -369,11 +375,13 @@ class User extends Model {
static async getUserByEmail(email) {
if (!email) return null
const cachedUser = userCache.getByEmail(email)
const normalizedEmail = email.toLowerCase()
const cachedUser = userCache.getByEmail(normalizedEmail)
if (cachedUser) return cachedUser
const user = await this.findOne({
where: sequelize.where(sequelize.fn('lower', sequelize.col('email')), email.toLowerCase()),
where: sequelize.where(sequelize.fn('LOWER', sequelize.col('email')), normalizedEmail),
include: this.sequelize.models.mediaProgress
})
@ -414,10 +422,24 @@ class User extends Model {
const cachedUser = userCache.getById(userId) || userCache.getByOldId(userId)
if (cachedUser) return cachedUser
const isUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userId)
if (isUuid) {
const byId = await this.findByPk(userId, {
include: this.sequelize.models.mediaProgress
})
if (byId) {
userCache.set(byId)
return byId
}
}
const oldIdMatcher =
this.sequelize.getDialect() === 'postgres'
? sequelize.where(sequelize.literal(`extradata#>>'{oldUserId}'`), userId)
: { 'extraData.oldUserId': userId }
const user = await this.findOne({
where: {
[sequelize.Op.or]: [{ id: userId }, { 'extraData.oldUserId': userId }]
},
where: oldIdMatcher,
include: this.sequelize.models.mediaProgress
})

6
server/objects/Backup.js
View file

@ -62,9 +62,9 @@ class Backup {
}
}
setData(backupDirPath) {
setData(backupDirPath, dialect = 'sqlite') {
this.id = date.format(new Date(), 'YYYY-MM-DD[T]HHmm')
this.key = 'sqlite'
this.key = dialect
this.datePretty = date.format(new Date(), 'ddd, MMM D YYYY HH:mm')
this.backupDirPath = backupDirPath
@ -78,4 +78,4 @@ class Backup {
this.createdAt = Date.now()
}
}
module.exports = Backup
module.exports = Backup

122
server/routers/ApiRouter.js
View file

@ -527,54 +527,128 @@ class ApiRouter {
return userSessions.sort((a, b) => b.updatedAt - a.updatedAt)
}
async getUserListeningSessionsPageHelper(userId, page = 0, itemsPerPage = 10, mediaItemId = null) {
const where = { userId }
if (mediaItemId) where.mediaItemId = mediaItemId
const start = page * itemsPerPage
const [total, sessions] = await Promise.all([
Database.countPlaybackSessions(where),
Database.getPlaybackSessions(where, {
limit: itemsPerPage,
offset: start,
order: [['updatedAt', 'DESC']]
})
])
return {
total,
numPages: Math.ceil(total / itemsPerPage),
page,
itemsPerPage,
sessions
}
}
async getUserItemListeningSessionsHelper(userId, mediaItemId) {
const userSessions = await Database.getPlaybackSessions({ userId, mediaItemId })
return userSessions.sort((a, b) => b.updatedAt - a.updatedAt)
}
async getUserListeningStatsHelpers(userId) {
async getUserListeningStatsHelpers(userId, options = {}) {
const startedAt = Date.now()
const today = date.format(new Date(), 'YYYY-MM-DD')
const includeItems = options.includeItems !== false
const includeRecentSessions = options.includeRecentSessions !== false
const getSessionField = (session, key) => {
if (!session || typeof session !== 'object') return undefined
if (session[key] !== undefined) return session[key]
const lowerKey = key.toLowerCase()
if (session[lowerKey] !== undefined) return session[lowerKey]
return undefined
}
const getSessionObjectField = (session, key) => {
const value = getSessionField(session, key)
if (!value) return null
if (typeof value === 'string') {
try {
return JSON.parse(value)
} catch (error) {
return null
}
}
return value
}
const tasks = [Database.getPlaybackSessionsForStats({ userId })]
if (includeRecentSessions) {
tasks.push(
Database.getPlaybackSessions({ userId }, {
limit: 10,
offset: 0,
order: [['updatedAt', 'DESC']]
})
)
}
const [listeningSessions, recentSessions = []] = await Promise.all(tasks)
const listeningSessions = await this.getUserListeningSessionsHelper(userId)
const listeningStats = {
totalTime: 0,
items: {},
days: {},
dayOfWeek: {},
today: 0,
recentSessions: listeningSessions.slice(0, 10)
today: 0
}
if (includeItems) listeningStats.items = {}
if (includeRecentSessions) listeningStats.recentSessions = recentSessions
listeningSessions.forEach((s) => {
let sessionTimeListening = s.timeListening
if (typeof sessionTimeListening == 'string') {
sessionTimeListening = Number(sessionTimeListening)
}
const extraData = getSessionObjectField(s, 'extraData')
const libraryItemId = extraData?.libraryItemId || extraData?.libraryitemid || null
const sessionDate = getSessionField(s, 'date')
const sessionDayOfWeek = getSessionField(s, 'dayOfWeek')
const sessionUpdatedAt = getSessionField(s, 'updatedAt')
const sessionMediaMetadata = getSessionObjectField(s, 'mediaMetadata')
const numericListening = Number(getSessionField(s, 'timeListening'))
const sessionTimeListening = Number.isFinite(numericListening)
? numericListening
: 0
if (s.dayOfWeek) {
if (!listeningStats.dayOfWeek[s.dayOfWeek]) listeningStats.dayOfWeek[s.dayOfWeek] = 0
listeningStats.dayOfWeek[s.dayOfWeek] += sessionTimeListening
if (sessionDayOfWeek) {
if (!listeningStats.dayOfWeek[sessionDayOfWeek]) listeningStats.dayOfWeek[sessionDayOfWeek] = 0
listeningStats.dayOfWeek[sessionDayOfWeek] += sessionTimeListening
}
if (s.date && sessionTimeListening > 0) {
if (!listeningStats.days[s.date]) listeningStats.days[s.date] = 0
listeningStats.days[s.date] += sessionTimeListening
if (sessionDate && sessionTimeListening > 0) {
if (!listeningStats.days[sessionDate]) listeningStats.days[sessionDate] = 0
listeningStats.days[sessionDate] += sessionTimeListening
if (s.date === today) {
if (sessionDate === today) {
listeningStats.today += sessionTimeListening
}
}
if (!listeningStats.items[s.libraryItemId]) {
listeningStats.items[s.libraryItemId] = {
id: s.libraryItemId,
timeListening: sessionTimeListening,
mediaMetadata: s.mediaMetadata,
lastUpdate: s.lastUpdate
if (!libraryItemId) {
listeningStats.totalTime += sessionTimeListening
return
}
if (includeItems) {
if (!listeningStats.items[libraryItemId]) {
listeningStats.items[libraryItemId] = {
id: libraryItemId,
timeListening: sessionTimeListening,
mediaMetadata: sessionMediaMetadata,
lastUpdate: sessionUpdatedAt
}
} else {
listeningStats.items[libraryItemId].timeListening += sessionTimeListening
}
} else {
listeningStats.items[s.libraryItemId].timeListening += sessionTimeListening
}
listeningStats.totalTime += sessionTimeListening
})
Logger.debug(
`[ApiRouter] Listening stats for user "${userId}" aggregated ${listeningSessions.length} sessions in ${Date.now() - startedAt}ms includeItems=${includeItems} includeRecentSessions=${includeRecentSessions}`
)
return listeningStats
}
}

7
server/scanner/LibraryScanner.js
View file

@ -14,6 +14,7 @@ const LibraryItemScanner = require('./LibraryItemScanner')
const LibraryScan = require('./LibraryScan')
const LibraryItemScanData = require('./LibraryItemScanData')
const Task = require('../objects/Task')
const { isPostgres, jsonArrayExpand } = require('../utils/sqlDialectHelpers')
class LibraryScanner {
constructor() {
@ -676,12 +677,16 @@ async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingle
// check if it was moved from another folder by comparing the ino to the library files
const ino = await fileUtils.getIno(fullPath)
if (!ino) return null
const inodeMatchQuery = isPostgres(Database.sequelize)
? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{ino}' = :inode)`
: `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)`
const existingLibraryItem = await Database.libraryItemModel.findOneExpanded(
[
{
libraryId: libraryId
},
sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), {
sequelize.where(sequelize.literal(inodeMatchQuery), {
[sequelize.Op.gt]: 0
})
],

515
server/scripts/migrateSqliteToPostgres.js Normal file
View file

@ -0,0 +1,515 @@
#!/usr/bin/env node
const sqlite3 = require('sqlite3')
const { Client } = require('pg')
const SQLITE_PATH = process.env.SQLITE_PATH || '/config/absdatabase.sqlite'
const DATABASE_URL = process.env.DATABASE_URL
const PG_SCHEMA = process.env.PG_SCHEMA || 'public'
const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500)
const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true'
const ALLOW_DESTRUCTIVE_TARGET = String(process.env.ALLOW_DESTRUCTIVE_TARGET || 'false').toLowerCase() === 'true'
const integerBounds = {
smallint: { min: -32768n, max: 32767n },
integer: { min: -2147483648n, max: 2147483647n },
bigint: { min: -9223372036854775808n, max: 9223372036854775807n }
}
const preferredOrder = [
'migrationsMeta',
'SequelizeMeta',
'settings',
'users',
'apiKeys',
'sessions',
'libraries',
'libraryFolders',
'authors',
'series',
'books',
'podcasts',
'podcastEpisodes',
'libraryItems',
'bookAuthors',
'bookSeries',
'collections',
'collectionBooks',
'playlists',
'playlistMediaItems',
'mediaProgresses',
'devices',
'playbackSessions',
'feeds',
'feedEpisodes',
'mediaItemShares',
'customMetadataProviders'
]
function quoteIdent(identifier) {
return `"${String(identifier).replace(/"/g, '""')}"`
}
function openSqlite(filePath) {
return new Promise((resolve, reject) => {
const db = new sqlite3.Database(filePath, sqlite3.OPEN_READONLY, (err) => {
if (err) return reject(err)
resolve(db)
})
})
}
function sqliteAll(db, sql, params = []) {
return new Promise((resolve, reject) => {
db.all(sql, params, (err, rows) => {
if (err) return reject(err)
resolve(rows)
})
})
}
function sqliteGet(db, sql, params = []) {
return new Promise((resolve, reject) => {
db.get(sql, params, (err, row) => {
if (err) return reject(err)
resolve(row)
})
})
}
async function findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) {
const issues = []
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
const pgColumns = pgColumnsByTable.get(postgresTable)
if (!pgColumns) continue
for (const column of pgColumns.values()) {
if (column.data_type !== 'character varying' || !column.character_maximum_length) continue
const sqliteColumn = column.column_name
const maxLength = Number(column.character_maximum_length)
const quotedTable = quoteIdent(sqliteTable)
const quotedColumn = quoteIdent(sqliteColumn)
try {
const maxLengthRow = await sqliteGet(
sqliteDb,
`SELECT MAX(LENGTH(${quotedColumn})) AS maxLength FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`
)
const actualMaxLength = Number(maxLengthRow?.maxLength || 0)
if (actualMaxLength <= maxLength) continue
const overCountRow = await sqliteGet(
sqliteDb,
`SELECT COUNT(*) AS count FROM ${quotedTable} WHERE LENGTH(${quotedColumn}) > ?`,
[maxLength]
)
issues.push({
sqliteTable,
sqliteColumn,
postgresTable,
postgresColumn: column.column_name,
maxLength,
actualMaxLength,
overCount: Number(overCountRow?.count || 0)
})
} catch (error) {
// Ignore columns missing in sqlite source table
}
}
}
return issues
}
function parseIntegerValue(value) {
if (typeof value === 'number') {
if (!Number.isSafeInteger(value)) return null
return BigInt(value)
}
if (typeof value === 'string') {
const trimmed = value.trim()
if (!trimmed || !/^-?\d+$/.test(trimmed)) return null
try {
return BigInt(trimmed)
} catch (error) {
return null
}
}
return null
}
function isIntegerCompatible(value, dataType = 'bigint') {
if (value === null || value === undefined) return true
const parsed = parseIntegerValue(value)
if (parsed === null) return false
const bounds = integerBounds[dataType]
if (!bounds) return false
return parsed >= bounds.min && parsed <= bounds.max
}
async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) {
const issues = []
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
const pgColumns = pgColumnsByTable.get(postgresTable)
if (!pgColumns) continue
for (const column of pgColumns.values()) {
const dataType = column.data_type
if (dataType !== 'smallint' && dataType !== 'integer' && dataType !== 'bigint') continue
const sqliteColumn = column.column_name
const quotedTable = quoteIdent(sqliteTable)
const quotedColumn = quoteIdent(sqliteColumn)
let rows
try {
rows = await sqliteAll(sqliteDb, `SELECT ${quotedColumn} AS value FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`)
} catch (error) {
// Ignore columns missing in sqlite source table
continue
}
let badCount = 0
let sampleValue = null
for (const row of rows) {
if (!isIntegerCompatible(row.value, dataType)) {
badCount += 1
if (sampleValue === null) sampleValue = row.value
}
}
if (badCount > 0) {
issues.push({
sqliteTable,
sqliteColumn,
postgresTable,
postgresColumn: column.column_name,
postgresType: dataType,
badCount,
sampleValue
})
}
}
}
return issues
}
function normalizeBoolean(value) {
if (value === null || value === undefined) return null
if (typeof value === 'boolean') return value
if (typeof value === 'number') return value !== 0
if (typeof value === 'string') {
const v = value.trim().toLowerCase()
return v === 'true' || v === '1' || v === 't'
}
return !!value
}
function normalizeJson(value) {
if (value === null || value === undefined || value === '') return null
if (typeof value === 'object') return JSON.stringify(value)
const textValue = String(value)
try {
const parsed = JSON.parse(textValue)
if (typeof parsed === 'string') {
try {
return JSON.stringify(JSON.parse(parsed))
} catch (error) {
return JSON.stringify(parsed)
}
}
return JSON.stringify(parsed)
} catch (error) {
// Keep non-JSON payloads as JSON string values so inserts remain valid JSON.
return JSON.stringify(textValue)
}
}
function convertValue(value, pgColumn) {
if (!pgColumn) return value
const dataType = pgColumn.data_type
const udtName = pgColumn.udt_name
if (dataType === 'boolean') {
return normalizeBoolean(value)
}
if (dataType === 'json' || dataType === 'jsonb' || udtName === 'json' || udtName === 'jsonb') {
return normalizeJson(value)
}
if ((dataType === 'smallint' || dataType === 'integer' || dataType === 'bigint') && value !== null && value !== undefined) {
if (!isIntegerCompatible(value, dataType)) return value
if (dataType === 'bigint' && typeof value === 'string') {
return value.trim()
}
if (typeof value === 'number') return value
return Number(value)
}
return value
}
function getOverlongColumns(row, insertColumns) {
const overlong = []
for (const column of insertColumns) {
const maxLength = column.metadata.character_maximum_length
if (!maxLength) continue
const value = row[column.sqliteColumn]
if (value === null || value === undefined) continue
const length = String(value).length
if (length > maxLength) {
overlong.push({
sqliteColumn: column.sqliteColumn,
postgresColumn: column.postgresColumn,
maxLength,
actualLength: length
})
}
}
return overlong
}
async function main() {
if (!DATABASE_URL) {
throw new Error('DATABASE_URL is required')
}
console.log(`[migrate] sqlite source: ${SQLITE_PATH}`)
console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`)
console.log(`[migrate] dry run: ${DRY_RUN}`)
const sqliteDb = await openSqlite(SQLITE_PATH)
const pg = new Client({ connectionString: DATABASE_URL })
await pg.connect()
try {
const sqliteTablesRows = await sqliteAll(
sqliteDb,
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
)
const sqliteTables = sqliteTablesRows.map((row) => row.name)
const pgTablesRows = await pg.query(
`SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`,
[PG_SCHEMA]
)
const pgTablesByLowerName = new Map(pgTablesRows.rows.map((row) => [row.table_name.toLowerCase(), row.table_name]))
const tablesToMigrate = sqliteTables
.map((sqliteTable) => {
const postgresTable = pgTablesByLowerName.get(sqliteTable.toLowerCase())
if (!postgresTable) return null
return {
sqliteTable,
postgresTable
}
})
.filter(Boolean)
tablesToMigrate.sort((a, b) => {
const ai = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === a.sqliteTable.toLowerCase())
const bi = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === b.sqliteTable.toLowerCase())
if (ai === -1 && bi === -1) return a.sqliteTable.localeCompare(b.sqliteTable)
if (ai === -1) return 1
if (bi === -1) return -1
return ai - bi
})
if (!tablesToMigrate.length) {
throw new Error('No overlapping tables found between SQLite and PostgreSQL')
}
console.log(`[migrate] tables to migrate: ${tablesToMigrate.map((table) => table.sqliteTable).join(', ')}`)
const pgColumnsByTable = new Map()
for (const { postgresTable } of tablesToMigrate) {
const columnsResult = await pg.query(
`SELECT column_name, data_type, udt_name, character_maximum_length FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`,
[PG_SCHEMA, postgresTable]
)
const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column]))
pgColumnsByTable.set(postgresTable, columnMap)
}
const overlongVarcharIssues = await findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable)
const integerTypeIssues = await findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable)
if (overlongVarcharIssues.length) {
console.error('[migrate] overlong source values detected for varchar columns:')
overlongVarcharIssues.forEach((issue) => {
console.error(
`[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` +
`(max=${issue.maxLength}, actualMax=${issue.actualMaxLength}, overRows=${issue.overCount})`
)
})
throw new Error('Migration aborted to prevent truncation/data loss. Widen target column types first.')
}
if (integerTypeIssues.length) {
console.error('[migrate] non-integer source values detected for integer columns:')
integerTypeIssues.forEach((issue) => {
console.error(
`[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` +
`(type=${issue.postgresType}, badRows=${issue.badCount}, sample=${JSON.stringify(issue.sampleValue)})`
)
})
throw new Error('Migration aborted to prevent numeric precision loss. Widen target numeric column types first.')
}
if (!DRY_RUN) {
if (!ALLOW_DESTRUCTIVE_TARGET) {
throw new Error('Migration writes are destructive. Set ALLOW_DESTRUCTIVE_TARGET=true after confirming the target database can be truncated.')
}
await pg.query('BEGIN')
await pg.query('SET session_replication_role = replica')
const truncateList = tablesToMigrate.map(({ postgresTable }) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`).join(', ')
await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`)
console.log('[migrate] truncated target tables')
}
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(sqliteTable)}`)
const pgColumns = pgColumnsByTable.get(postgresTable)
const insertColumns = rows.length
? Object.keys(rows[0])
.map((sqliteColumn) => {
const pgColumn = pgColumns.get(sqliteColumn.toLowerCase())
if (!pgColumn) return null
return {
sqliteColumn,
postgresColumn: pgColumn.column_name,
metadata: pgColumn
}
})
.filter(Boolean)
: []
if (!rows.length || !insertColumns.length) {
console.log(`[migrate] ${sqliteTable}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`)
continue
}
if (!DRY_RUN) {
for (let offset = 0; offset < rows.length; offset += BATCH_SIZE) {
const batchRows = rows.slice(offset, offset + BATCH_SIZE)
const valuesSql = []
const params = []
let paramIndex = 1
for (const row of batchRows) {
const placeholders = []
for (const column of insertColumns) {
params.push(convertValue(row[column.sqliteColumn], column.metadata))
placeholders.push(`$${paramIndex++}`)
}
valuesSql.push(`(${placeholders.join(', ')})`)
}
const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}`
try {
await pg.query('SAVEPOINT migrate_batch')
await pg.query(insertSql, params)
await pg.query('RELEASE SAVEPOINT migrate_batch')
} catch (error) {
await pg.query('ROLLBACK TO SAVEPOINT migrate_batch')
console.error(`[migrate] batch insert failed for ${sqliteTable} (offset=${offset}, size=${batchRows.length}): ${error.message}`)
for (let rowIndex = 0; rowIndex < batchRows.length; rowIndex++) {
const row = batchRows[rowIndex]
const singleRowParams = insertColumns.map((column) => convertValue(row[column.sqliteColumn], column.metadata))
const singleRowInsertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES (${singleRowParams.map((_, index) => `$${index + 1}`).join(', ')})`
try {
await pg.query('SAVEPOINT migrate_row')
await pg.query(singleRowInsertSql, singleRowParams)
await pg.query('RELEASE SAVEPOINT migrate_row')
} catch (rowError) {
await pg.query('ROLLBACK TO SAVEPOINT migrate_row')
const overlongColumns = getOverlongColumns(row, insertColumns)
if (overlongColumns.length) {
overlongColumns.forEach((column) => {
console.error(
`[migrate] overlong value in ${sqliteTable}.${column.sqliteColumn} -> ${postgresTable}.${column.postgresColumn} ` +
`(length=${column.actualLength}, max=${column.maxLength})`
)
})
}
throw rowError
}
}
}
}
}
console.log(`[migrate] ${sqliteTable}: ${rows.length} rows`)
}
if (!DRY_RUN) {
await pg.query('SET session_replication_role = DEFAULT')
await pg.query('COMMIT')
console.log('[migrate] migration transaction committed')
}
const parity = []
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(sqliteTable)}`)
const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`)
parity.push({
table: sqliteTable,
sqliteCount: Number(sqliteCountRow.count || 0),
postgresCount: Number(pgCountResult.rows[0].count || 0)
})
}
const mismatches = parity.filter((row) => row.sqliteCount !== row.postgresCount)
if (mismatches.length) {
console.error('[migrate] row-count mismatches detected:')
mismatches.forEach((row) => {
console.error(`[migrate] ${row.table}: sqlite=${row.sqliteCount} postgres=${row.postgresCount}`)
})
process.exitCode = 2
} else {
console.log('[migrate] parity check passed for all migrated tables')
}
} finally {
sqliteDb.close()
await pg.end()
}
}
if (require.main === module) {
main().catch((error) => {
console.error('[migrate] failed:', error)
process.exit(1)
})
}
module.exports = {
normalizeJson,
isIntegerCompatible,
convertValue,
findOverlongVarcharValues,
findIntegerTypeIssues,
quoteIdent
}

356
server/utils/queries/libraryFilters.js
View file

@ -5,10 +5,22 @@ const libraryItemsBookFilters = require('./libraryItemsBookFilters')
const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters')
const { createNewSortInstance } = require('../../libs/fastSort')
const { profile } = require('../../utils/profiler')
const { booleanLiteral, jsonArrayContainsAny } = require('../sqlDialectHelpers')
const naturalSort = createNewSortInstance({
comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
})
async function withShelfFallback(scope, fallbackValue, action) {
try {
return await action()
} catch (error) {
// Only postgres shelves fall back to empty results - on sqlite keep upstream behavior of surfacing the error
if (!Database.isPostgresDialect()) throw error
Logger.error(`[LibraryFilters] Failed to load ${scope}`, error)
return fallbackValue
}
}
module.exports = {
decode(text) {
return Buffer.from(decodeURIComponent(text), 'base64').toString()
@ -49,22 +61,24 @@ module.exports = {
* @returns {Promise<{ items:import('../../models/LibraryItem')[], count:number }>}
*/
async getMediaItemsInProgress(library, user, include, limit) {
if (library.isBook) {
const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'in-progress', 'progress', true, false, include, limit, 0, true)
return {
items: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
if (li.rssFeed) {
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
}
if (li.mediaItemShare) {
oldLibraryItem.mediaItemShare = li.mediaItemShare
}
return oldLibraryItem
}),
count
return withShelfFallback(`in-progress shelf for library "${library.id}"`, { items: [], count: 0 }, async () => {
if (library.isBook) {
const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'in-progress', 'progress', true, false, include, limit, 0, true)
return {
items: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
if (li.rssFeed) {
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
}
if (li.mediaItemShare) {
oldLibraryItem.mediaItemShare = li.mediaItemShare
}
return oldLibraryItem
}),
count
}
}
} else {
const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'in-progress', 'progress', true, limit, 0, true)
return {
count,
@ -74,7 +88,7 @@ module.exports = {
return oldLibraryItem
})
}
}
})
},
/**
@ -86,25 +100,27 @@ module.exports = {
* @returns {object} { libraryItems:LibraryItem[], count:number }
*/
async getLibraryItemsMostRecentlyAdded(library, user, include, limit) {
if (library.isBook) {
const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, false, include, limit, 0)
return {
libraryItems: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
if (li.rssFeed) {
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
}
if (li.size && !oldLibraryItem.media.size) {
oldLibraryItem.media.size = li.size
}
if (li.mediaItemShare) {
oldLibraryItem.mediaItemShare = li.mediaItemShare
}
return oldLibraryItem
}),
count
return withShelfFallback(`recent shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => {
if (library.isBook) {
const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, false, include, limit, 0)
return {
libraryItems: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
if (li.rssFeed) {
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
}
if (li.size && !oldLibraryItem.media.size) {
oldLibraryItem.media.size = li.size
}
if (li.mediaItemShare) {
oldLibraryItem.mediaItemShare = li.mediaItemShare
}
return oldLibraryItem
}),
count
}
}
} else {
const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, include, limit, 0)
return {
libraryItems: libraryItems.map((li) => {
@ -122,7 +138,7 @@ module.exports = {
}),
count
}
}
})
},
/**
@ -134,23 +150,25 @@ module.exports = {
* @returns {object} { libraryItems:LibraryItem[], count:number }
*/
async getLibraryItemsContinueSeries(library, user, include, limit) {
const { libraryItems, count } = await libraryItemsBookFilters.getContinueSeriesLibraryItems(library, user, include, limit, 0)
return {
libraryItems: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
if (li.rssFeed) {
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
}
if (li.series) {
oldLibraryItem.media.metadata.series = li.series
}
if (li.mediaItemShare) {
oldLibraryItem.mediaItemShare = li.mediaItemShare
}
return oldLibraryItem
}),
count
}
return withShelfFallback(`continue-series shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => {
const { libraryItems, count } = await libraryItemsBookFilters.getContinueSeriesLibraryItems(library, user, include, limit, 0)
return {
libraryItems: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
if (li.rssFeed) {
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
}
if (li.series) {
oldLibraryItem.media.metadata.series = li.series
}
if (li.mediaItemShare) {
oldLibraryItem.mediaItemShare = li.mediaItemShare
}
return oldLibraryItem
}),
count
}
})
},
/**
@ -163,22 +181,24 @@ module.exports = {
* @returns {Promise<{ items:oldLibraryItem[], count:number }>}
*/
async getMediaFinished(library, user, include, limit) {
if (library.isBook) {
const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'finished', 'progress', true, false, include, limit, 0)
return {
items: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
if (li.rssFeed) {
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
}
if (li.mediaItemShare) {
oldLibraryItem.mediaItemShare = li.mediaItemShare
}
return oldLibraryItem
}),
count
return withShelfFallback(`finished shelf for library "${library.id}"`, { items: [], count: 0 }, async () => {
if (library.isBook) {
const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'finished', 'progress', true, false, include, limit, 0)
return {
items: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
if (li.rssFeed) {
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
}
if (li.mediaItemShare) {
oldLibraryItem.mediaItemShare = li.mediaItemShare
}
return oldLibraryItem
}),
count
}
}
} else {
const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'finished', 'progress', true, limit, 0)
return {
count,
@ -188,7 +208,7 @@ module.exports = {
return oldLibraryItem
})
}
}
})
},
/**
@ -202,23 +222,24 @@ module.exports = {
async getSeriesMostRecentlyAdded(library, user, include, limit) {
if (!library.isBook) return { series: [], count: 0 }
const seriesIncludes = []
if (include.includes('rssfeed')) {
seriesIncludes.push({
model: Database.feedModel
})
}
const userPermissionBookWhere = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user)
const seriesWhere = [
{
libraryId: library.id,
createdAt: {
[Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
}
return withShelfFallback(`recent-series shelf for library "${library.id}"`, { series: [], count: 0 }, async () => {
const seriesIncludes = []
if (include.includes('rssfeed')) {
seriesIncludes.push({
model: Database.feedModel
})
}
]
const userPermissionBookWhere = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user)
const seriesWhere = [
{
libraryId: library.id,
createdAt: {
[Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
}
}
]
// Handle library setting to hide single book series
// TODO: Merge with existing query
@ -235,13 +256,13 @@ module.exports = {
if (userPermissionBookWhere.bookWhere.length) {
let attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id'
if (!user.canAccessExplicitContent) {
attrQuery += ' AND b.explicit = 0'
attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}`
}
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
if (user.permissions.selectedTagsNotAccessible) {
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0'
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0`
} else {
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0'
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0`
}
}
seriesWhere.push(
@ -251,29 +272,29 @@ module.exports = {
)
}
const { rows: series, count } = await Database.seriesModel.findAndCountAll({
where: seriesWhere,
limit,
offset: 0,
distinct: true,
subQuery: false,
replacements: userPermissionBookWhere.replacements,
include: [
{
model: Database.bookSeriesModel,
include: {
model: Database.bookModel,
where: userPermissionBookWhere.bookWhere,
const { rows: series, count } = await Database.seriesModel.findAndCountAll({
where: seriesWhere,
limit,
offset: 0,
distinct: true,
subQuery: false,
replacements: userPermissionBookWhere.replacements,
include: [
{
model: Database.bookSeriesModel,
include: {
model: Database.libraryItemModel
}
model: Database.bookModel,
where: userPermissionBookWhere.bookWhere,
include: {
model: Database.libraryItemModel
}
},
separate: true
},
separate: true
},
...seriesIncludes
],
order: [['createdAt', 'DESC']]
})
...seriesIncludes
],
order: [['createdAt', 'DESC']]
})
const allOldSeries = []
for (const s of series) {
@ -311,10 +332,11 @@ module.exports = {
allOldSeries.push(oldSeries)
}
return {
series: allOldSeries,
count
}
return {
series: allOldSeries,
count
}
})
},
/**
@ -329,37 +351,39 @@ module.exports = {
async getNewestAuthors(library, user, limit) {
if (library.mediaType !== 'book') return { authors: [], count: 0 }
const { bookWhere, replacements } = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user)
return withShelfFallback(`newest-authors shelf for library "${library.id}"`, { authors: [], count: 0 }, async () => {
const { bookWhere, replacements } = libraryItemsBookFilters.getUserPermissionBookWhereQuery(user)
const { rows: authors, count } = await Database.authorModel.findAndCountAll({
where: {
libraryId: library.id,
createdAt: {
[Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
}
},
replacements,
include: {
model: Database.bookModel,
attributes: ['id', 'tags', 'explicit'],
where: bookWhere,
required: true, // Must belong to a book
through: {
attributes: []
}
},
limit,
distinct: true,
order: [['createdAt', 'DESC']]
const { rows: authors, count } = await Database.authorModel.findAndCountAll({
where: {
libraryId: library.id,
createdAt: {
[Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
}
},
replacements,
include: {
model: Database.bookModel,
attributes: ['id', 'tags', 'explicit'],
where: bookWhere,
required: true, // Must belong to a book
through: {
attributes: []
}
},
limit,
distinct: true,
order: [['createdAt', 'DESC']]
})
return {
authors: authors.map((au) => {
const numBooks = au.books.length || 0
return au.toOldJSONExpanded(numBooks)
}),
count
}
})
return {
authors: authors.map((au) => {
const numBooks = au.books.length || 0
return au.toOldJSONExpanded(numBooks)
}),
count
}
},
/**
@ -373,20 +397,22 @@ module.exports = {
async getLibraryItemsToDiscover(library, user, include, limit) {
if (library.mediaType !== 'book') return { libraryItems: [], count: 0 }
const { libraryItems, count } = await libraryItemsBookFilters.getDiscoverLibraryItems(library.id, user, include, limit)
return {
libraryItems: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
if (li.rssFeed) {
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
}
if (li.mediaItemShare) {
oldLibraryItem.mediaItemShare = li.mediaItemShare
}
return oldLibraryItem
}),
count
}
return withShelfFallback(`discover shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => {
const { libraryItems, count } = await libraryItemsBookFilters.getDiscoverLibraryItems(library.id, user, include, limit)
return {
libraryItems: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
if (li.rssFeed) {
oldLibraryItem.rssFeed = li.rssFeed.toOldJSONMinified()
}
if (li.mediaItemShare) {
oldLibraryItem.mediaItemShare = li.mediaItemShare
}
return oldLibraryItem
}),
count
}
})
},
/**
@ -399,15 +425,17 @@ module.exports = {
async getNewestPodcastEpisodes(library, user, limit) {
if (library.mediaType !== 'podcast') return { libraryItems: [], count: 0 }
const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'recent', null, 'createdAt', true, limit, 0)
return {
count,
libraryItems: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
oldLibraryItem.recentEpisode = li.recentEpisode
return oldLibraryItem
})
}
return withShelfFallback(`newest-podcast-episodes shelf for library "${library.id}"`, { libraryItems: [], count: 0 }, async () => {
const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'recent', null, 'createdAt', true, limit, 0)
return {
count,
libraryItems: libraryItems.map((li) => {
const oldLibraryItem = li.toOldJSONMinified()
oldLibraryItem.recentEpisode = li.recentEpisode
return oldLibraryItem
})
}
})
},
/**

11
server/utils/queries/libraryItemFilters.js
View file

@ -2,6 +2,7 @@ const Sequelize = require('sequelize')
const Database = require('../../Database')
const libraryItemsBookFilters = require('./libraryItemsBookFilters')
const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters')
const { jsonArrayContainsAny } = require('../sqlDialectHelpers')
module.exports = {
/**
@ -12,7 +13,7 @@ module.exports = {
async getAllLibraryItemsWithTags(tags) {
const libraryItems = []
const booksWithTag = await Database.bookModel.findAll({
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), {
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), {
[Sequelize.Op.gte]: 1
}),
replacements: {
@ -46,7 +47,7 @@ module.exports = {
libraryItems.push(libraryItem)
}
const podcastsWithTag = await Database.podcastModel.findAll({
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), {
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), {
[Sequelize.Op.gte]: 1
}),
replacements: {
@ -77,7 +78,7 @@ module.exports = {
async getAllLibraryItemsWithGenres(genres) {
const libraryItems = []
const booksWithGenre = await Database.bookModel.findAll({
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), {
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), {
[Sequelize.Op.gte]: 1
}),
replacements: {
@ -107,7 +108,7 @@ module.exports = {
libraryItems.push(libraryItem)
}
const podcastsWithGenre = await Database.podcastModel.findAll({
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), {
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), {
[Sequelize.Op.gte]: 1
}),
replacements: {
@ -138,7 +139,7 @@ module.exports = {
async getAllLibraryItemsWithNarrators(narrators) {
const libraryItems = []
const booksWithGenre = await Database.bookModel.findAll({
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(narrators) WHERE json_valid(narrators) AND json_each.value IN (:narrators))`), {
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('narrators', 'narrators', Database.sequelize)), {
[Sequelize.Op.gte]: 1
}),
replacements: {

151
server/utils/queries/libraryItemsBookFilters.js
View file

@ -6,6 +6,7 @@ const authorFilters = require('./authorFilters')
const ShareManager = require('../../managers/ShareManager')
const { profile } = require('../profiler')
const stringifySequelizeQuery = require('../stringifySequelizeQuery')
const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand, safeTextToDoubleExpression, safeTextToIntegerExpression } = require('../sqlDialectHelpers')
const countCache = new Map()
module.exports = {
@ -27,10 +28,10 @@ module.exports = {
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
replacements['userTagsSelected'] = user.permissions.itemTagsSelected
if (user.permissions.selectedTagsNotAccessible) {
bookWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0))
bookWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0))
} else {
bookWhere.push(
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), {
Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), {
[Sequelize.Op.gte]: 1
})
)
@ -189,7 +190,7 @@ module.exports = {
} else if (group === 'explicit') {
mediaWhere['explicit'] = true
} else if (['genres', 'tags', 'narrators'].includes(group)) {
mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), {
mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), {
[Sequelize.Op.gte]: 1
})
replacements.filterValue = value
@ -236,7 +237,7 @@ module.exports = {
} else if (group === 'publishedDecades') {
const startYear = parseInt(value)
const endYear = parseInt(value, 10) + 9
mediaWhere = Sequelize.where(Sequelize.literal('CAST(publishedYear AS INTEGER)'), {
mediaWhere = Sequelize.where(Sequelize.literal(safeTextToIntegerExpression('publishedYear', Database.sequelize)), {
[Sequelize.Op.between]: [startYear, endYear]
})
}
@ -256,9 +257,9 @@ module.exports = {
const getTitleOrder = () => {
if (global.ServerSettings.sortingIgnorePrefix) {
return [Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]
return [Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]
} else {
return [Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]
return [Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]
}
}
@ -273,21 +274,22 @@ module.exports = {
} else if (sortBy === 'media.duration') {
return [['duration', dir]]
} else if (sortBy === 'media.metadata.publishedYear') {
return [[Sequelize.literal(`CAST(\`book\`.\`publishedYear\` AS INTEGER)`), dir]]
return [[Sequelize.literal(safeTextToIntegerExpression('book.publishedYear', Database.sequelize)), dir]]
} else if (sortBy === 'media.metadata.authorNameLF') {
// Sort by author name last first, secondary sort by title
return [[Sequelize.literal('`libraryItem`.`authorNamesLastFirst` COLLATE NOCASE'), dir], getTitleOrder()]
return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesLastFirst', Database.sequelize)), dir], getTitleOrder()]
} else if (sortBy === 'media.metadata.authorName') {
// Sort by author name first last, secondary sort by title
return [[Sequelize.literal('`libraryItem`.`authorNamesFirstLast` COLLATE NOCASE'), dir], getTitleOrder()]
return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesFirstLast', Database.sequelize)), dir], getTitleOrder()]
} else if (sortBy === 'media.metadata.title') {
if (collapseseries) {
return [[Sequelize.literal('display_title COLLATE NOCASE'), dir]]
return [[Sequelize.literal(noCaseSortExpression('display_title', Database.sequelize)), dir]]
}
return [getTitleOrder()]
} else if (sortBy === 'sequence') {
const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'
return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]]
const sequenceColumn = Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`'
return [[Sequelize.literal(`${safeTextToDoubleExpression(sequenceColumn, Database.sequelize)} ${nullDir}`)]]
} else if (sortBy === 'progress') {
return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]]
} else if (sortBy === 'progress.createdAt') {
@ -326,7 +328,9 @@ module.exports = {
required: true
}
],
order: [Sequelize.literal('CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]
order: [
Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"books->bookSeries"."sequence"' : '`books.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`)
]
})
const bookSeriesToInclude = []
const booksToInclude = []
@ -358,26 +362,32 @@ module.exports = {
async findAndCountAll(findOptions, limit, offset, useCountCache) {
const model = Database.bookModel
if (useCountCache) {
const countCacheKey = stringifySequelizeQuery(findOptions)
Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`)
if (!countCache.has(countCacheKey)) {
const count = await model.count(findOptions)
countCache.set(countCacheKey, count)
}
findOptions.limit = limit || null
findOptions.offset = offset
const rows = await model.findAll(findOptions)
return { rows, count: countCache.get(countCacheKey) }
}
findOptions.limit = limit || null
findOptions.offset = offset
return await model.findAndCountAll(findOptions)
try {
if (useCountCache) {
const countCacheKey = stringifySequelizeQuery(findOptions)
Logger.debug(`[LibraryItemsBookFilters] countCacheKey: ${countCacheKey}`)
if (!countCache.has(countCacheKey)) {
const count = await model.count(findOptions)
countCache.set(countCacheKey, count)
}
const rows = await model.findAll(findOptions)
return { rows, count: countCache.get(countCacheKey) }
}
return await model.findAndCountAll(findOptions)
} catch (error) {
Logger.error(`[LibraryItemsBookFilters] findAndCountAll failed: ${error.message}`)
try {
Logger.error(`[LibraryItemsBookFilters] findAndCountAll query: ${stringifySequelizeQuery(findOptions)}`)
} catch (stringifyError) {
Logger.error(`[LibraryItemsBookFilters] failed to stringify query: ${stringifyError.message}`)
}
throw error
}
},
/**
@ -455,12 +465,28 @@ module.exports = {
})
} else if (filterGroup === 'ebooks' && filterValue === 'supplementary') {
// TODO: Temp workaround for filtering supplementary ebook
libraryItemWhere['libraryFiles'] = {
[Sequelize.Op.substring]: `"isSupplementary":true`
if (Database.sequelize.getDialect() === 'postgres') {
libraryItemWhere[Sequelize.Op.and] = [
Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), {
[Sequelize.Op.like]: '%"isSupplementary":true%'
})
]
} else {
libraryItemWhere['libraryFiles'] = {
[Sequelize.Op.substring]: `"isSupplementary":true`
}
}
} else if (filterGroup === 'ebooks' && filterValue === 'no-supplementary') {
libraryItemWhere['libraryFiles'] = {
[Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`)
if (Database.sequelize.getDialect() === 'postgres') {
libraryItemWhere[Sequelize.Op.and] = [
Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), {
[Sequelize.Op.notLike]: '%"isSupplementary":true%'
})
]
} else {
libraryItemWhere['libraryFiles'] = {
[Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`)
}
}
} else if (filterGroup === 'missing' && filterValue === 'authors') {
authorInclude = {
@ -502,7 +528,9 @@ module.exports = {
})
if (sortBy !== 'sequence') {
// Secondary sort by sequence
sortOrder.push([Sequelize.literal('CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')])
sortOrder.push([
Sequelize.literal(`${safeTextToDoubleExpression(Database.sequelize.getDialect() === 'postgres' ? '"series->bookSeries"."sequence"' : '`series.bookSeries.sequence`', Database.sequelize)} ASC NULLS LAST`)
])
}
} else if (filterGroup === 'issues') {
libraryItemWhere[Sequelize.Op.or] = [
@ -594,10 +622,32 @@ module.exports = {
// When collapsing series and sorting by title then use the series name instead of the book title
// for this set an attribute "display_title" to use in sorting
const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.title' : '`libraryItem`.`title`'
const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? 'libraryItem.titleIgnorePrefix' : '`libraryItem`.`titleIgnorePrefix`'
const fallbackFn = coalesceFunctionName(Database.sequelize)
const includedBookSeriesIds = bookSeriesToInclude.map((v) => Database.sequelize.escape(v.id)).join(', ')
const collapseSeriesSubqueryByName = includedBookSeriesIds
? `(SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))`
: 'NULL'
const collapseSeriesSubqueryByNameIgnorePrefix = includedBookSeriesIds
? `(SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${includedBookSeriesIds}))`
: 'NULL'
Logger.debug(
`[LibraryItemsBookFilters] collapse-series computed includeIds=${bookSeriesToInclude.length} excludeBooks=${booksToExclude.length} ` +
`filterGroup=${filterGroup || 'none'} filterValue=${filterValue || 'none'} sortBy=${sortBy}`
)
if (!includedBookSeriesIds) {
Logger.debug(
`[LibraryItemsBookFilters] collapse-series produced no include IDs; using library item title fallback ` +
`(libraryId=${libraryId}, filterGroup=${filterGroup || 'none'}, filterValue=${filterValue || 'none'}, sortBy=${sortBy})`
)
}
if (global.ServerSettings.sortingIgnorePrefix) {
bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`titleIgnorePrefix\`)`), 'display_title'])
bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByNameIgnorePrefix}, ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title'])
} else {
bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`title\`)`), 'display_title'])
bookAttributes.include.push([Sequelize.literal(`${fallbackFn}(${collapseSeriesSubqueryByName}, ${fallbackLibraryItemTitle})`), 'display_title'])
}
}
@ -716,13 +766,14 @@ module.exports = {
bookWhere.push(...userPermissionBookWhere.bookWhere)
let includeAttributes = [[Sequelize.literal('(SELECT max(mp.updatedAt) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id)'), 'recent_progress']]
let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = 0 OR mp.isFinished IS NULL)`
let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = ${booleanLiteral(false, Database.sequelize)} OR mp.isFinished IS NULL)`
if (library.settings.onlyShowLaterBooksInContinueSeries) {
const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as FLOAT) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = 1 AND mp.userId = :userId AND bs.seriesId = series.id)`
const safeSequenceExpr = safeTextToDoubleExpression('bs.sequence', Database.sequelize)
const maxSequenceQuery = `(SELECT max(${safeSequenceExpr}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)`
includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence'])
booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as FLOAT) > ${maxSequenceQuery}`
booksNotFinishedQuery = booksNotFinishedQuery + ` AND ${safeSequenceExpr} > ${maxSequenceQuery}`
}
const { rows: series, count } = await Database.seriesModel.findAndCountAll({
@ -735,7 +786,7 @@ module.exports = {
},
// TODO: Simplify queries
// Has at least 1 book finished
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = 1)`), {
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)})`), {
[Sequelize.Op.gte]: 1
}),
// Has at least 1 book not finished (that has a sequence number higher than the highest already read, if library config is toggled)
@ -743,7 +794,7 @@ module.exports = {
[Sequelize.Op.gte]: 1
}),
// Has no books in progress
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = 0 AND mp.currentTime > 0)`), 0)
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)} AND mp.currentTime > 0)`), 0)
],
attributes: {
include: includeAttributes
@ -757,10 +808,10 @@ module.exports = {
attributes: ['bookId', 'sequence'],
separate: true,
subQuery: false,
order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]],
order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]],
where: {
'$book.mediaProgresses.isFinished$': {
[Sequelize.Op.or]: [null, 0]
[Sequelize.Op.or]: [null, false]
}
},
include: {
@ -854,7 +905,7 @@ module.exports = {
{
libraryId
},
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = 1 OR mp.currentTime > 0))`), 0)
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0))`), 0)
],
replacements: {
userId: user.id,
@ -870,7 +921,7 @@ module.exports = {
model: Database.bookModel,
where: userPermissionBookWhere.bookWhere
},
order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]],
order: [[Sequelize.literal(`${safeTextToDoubleExpression('sequence', Database.sequelize)} ASC NULLS LAST`)]],
limit: 1
},
subQuery: false,
@ -891,7 +942,7 @@ module.exports = {
const discoverWhere = [
{
'$mediaProgresses.isFinished$': {
[Sequelize.Op.or]: [null, 0]
[Sequelize.Op.or]: [null, false]
},
'$mediaProgresses.currentTime$': {
[Sequelize.Op.or]: [null, 0]
@ -1174,7 +1225,7 @@ module.exports = {
// Search narrators
const narratorMatches = []
const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, ${jsonArrayExpand('b.narrators', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
replacements: {
libraryId: library.id,
limit,
@ -1191,7 +1242,7 @@ module.exports = {
// Search tags
const tagMatches = []
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.tags', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: {
libraryId: library.id,
limit,
@ -1208,7 +1259,7 @@ module.exports = {
// Search genres
const genreMatches = []
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: {
libraryId: library.id,
limit,
@ -1288,7 +1339,7 @@ module.exports = {
*/
async getGenresWithCount(libraryId) {
const genres = []
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
replacements: {
libraryId
},

25
server/utils/queries/libraryItemsPodcastFilters.js
View file

@ -3,6 +3,7 @@ const Database = require('../../Database')
const Logger = require('../../Logger')
const { profile } = require('../../utils/profiler')
const stringifySequelizeQuery = require('../stringifySequelizeQuery')
const { jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression, jsonArrayExpand, jsonPathNumber } = require('../sqlDialectHelpers')
const countCache = new Map()
@ -24,10 +25,10 @@ module.exports = {
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
replacements['userTagsSelected'] = user.permissions.itemTagsSelected
if (user.permissions.selectedTagsNotAccessible) {
podcastWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0))
podcastWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0))
} else {
podcastWhere.push(
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), {
Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), {
[Sequelize.Op.gte]: 1
})
)
@ -53,7 +54,7 @@ module.exports = {
const replacements = {}
if (['genres', 'tags'].includes(group)) {
mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), {
mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), {
[Sequelize.Op.gte]: 1
})
replacements.filterValue = value
@ -87,12 +88,12 @@ module.exports = {
return [[Sequelize.literal('libraryItem.mtime'), dir]]
} else if (sortBy === 'media.metadata.author') {
const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'
return [[Sequelize.literal(`\`podcast\`.\`author\` COLLATE NOCASE ${nullDir}`)]]
return [[Sequelize.literal(`${noCaseSortExpression('podcast.author', Database.sequelize)} ${nullDir}`)]]
} else if (sortBy === 'media.metadata.title') {
if (global.ServerSettings.sortingIgnorePrefix) {
return [[Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]]
return [[Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]]
} else {
return [[Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]]
return [[Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]]
}
} else if (sortBy === 'media.numTracks') {
return [['numEpisodes', dir]]
@ -455,7 +456,7 @@ module.exports = {
// Search tags
const tagMatches = []
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.tags', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: {
libraryId: library.id,
limit,
@ -472,7 +473,7 @@ module.exports = {
// Search genres
const genreMatches = []
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: {
libraryId: library.id,
limit,
@ -563,12 +564,12 @@ module.exports = {
* @returns {Promise<{ totalSize:number, totalDuration:number, numAudioFiles:number, totalItems:number}>}
*/
async getPodcastLibraryStats(libraryId) {
const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = "podcast" AND li.libraryId = :libraryId;`, {
const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = 'podcast' AND li.libraryId = :libraryId;`, {
replacements: {
libraryId
}
})
const [statResults] = await Database.sequelize.query(`SELECT SUM(json_extract(pe.audioFile, '$.duration')) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, {
const [statResults] = await Database.sequelize.query(`SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, {
replacements: {
libraryId
}
@ -588,7 +589,7 @@ module.exports = {
*/
async getGenresWithCount(libraryId) {
const genres = []
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
replacements: {
libraryId
},
@ -611,7 +612,7 @@ module.exports = {
*/
async getLongestPodcasts(libraryId, limit) {
const podcasts = await Database.podcastModel.findAll({
attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(json_extract(pe.audioFile, '$.duration')) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']],
attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']],
include: {
model: Database.libraryItemModel,
attributes: ['id', 'libraryId'],

21
server/utils/queries/seriesFilters.js
View file

@ -2,6 +2,7 @@ const Sequelize = require('sequelize')
const Logger = require('../../Logger')
const Database = require('../../Database')
const libraryItemsBookFilters = require('./libraryItemsBookFilters')
const { booleanLiteral, jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression } = require('../sqlDialectHelpers')
module.exports = {
decode(text) {
@ -60,7 +61,7 @@ module.exports = {
// TODO: Simplify and break-out
let attrQuery = null
if (['genres', 'tags', 'narrators'].includes(filterGroup)) {
attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (SELECT count(*) FROM json_each(b.${filterGroup}) WHERE json_valid(b.${filterGroup}) AND json_each.value = :filterValue) > 0`
attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ${jsonArrayContainsValue(`b.${filterGroup}`, 'filterValue', Database.sequelize)} > 0`
userPermissionBookWhere.replacements.filterValue = filterValue
} else if (filterGroup === 'authors') {
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs, bookAuthors ba WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ba.bookId = b.id AND ba.authorId = :filterValue'
@ -73,18 +74,18 @@ module.exports = {
userPermissionBookWhere.replacements.filterValue = filterValue
} else if (filterGroup === 'progress') {
if (filterValue === 'not-finished') {
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)'
attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})`
userPermissionBookWhere.replacements.userId = user.id
} else if (filterValue === 'finished') {
const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)'
const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})`
seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0))
userPermissionBookWhere.replacements.userId = user.id
} else if (filterValue === 'not-started') {
const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = 1 OR mp.currentTime > 0)'
const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0)`
seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0))
userPermissionBookWhere.replacements.userId = user.id
} else if (filterValue === 'in-progress') {
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = 0'
attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)}`
userPermissionBookWhere.replacements.userId = user.id
}
}
@ -95,13 +96,13 @@ module.exports = {
if (!attrQuery) attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id'
if (!user.canAccessExplicitContent) {
attrQuery += ' AND b.explicit = 0'
attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}`
}
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
if (user.permissions.selectedTagsNotAccessible) {
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0'
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0`
} else {
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0'
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0`
}
}
}
@ -128,9 +129,9 @@ module.exports = {
order.push(['createdAt', dir])
} else if (sortBy === 'name') {
if (global.ServerSettings.sortingIgnorePrefix) {
order.push([Sequelize.literal('nameIgnorePrefix COLLATE NOCASE'), dir])
order.push([Sequelize.literal(noCaseSortExpression('nameIgnorePrefix', Database.sequelize)), dir])
} else {
order.push([Sequelize.literal('`series`.`name` COLLATE NOCASE'), dir])
order.push([Sequelize.literal(noCaseSortExpression('series.name', Database.sequelize)), dir])
}
} else if (sortBy === 'totalDuration') {
seriesAttributes.include.push([Sequelize.literal('(SELECT SUM(b.duration) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND b.id = bs.bookId)'), 'totalDuration'])

91
server/utils/sqlDialectHelpers.js Normal file
View file

@ -0,0 +1,91 @@
function getDialect(sequelize) {
if (!sequelize) return 'sqlite'
if (typeof sequelize.getDialect === 'function') return sequelize.getDialect()
return sequelize.dialect?.name || 'sqlite'
}
function isPostgres(sequelize) {
return getDialect(sequelize) === 'postgres'
}
function booleanLiteral(value, sequelize) {
if (isPostgres(sequelize)) return value ? 'TRUE' : 'FALSE'
return value ? '1' : '0'
}
function noCaseSortExpression(columnExpression, sequelize) {
if (isPostgres(sequelize)) return `LOWER(${columnExpression})`
return `${columnExpression} COLLATE NOCASE`
}
function coalesceFunctionName(sequelize) {
return isPostgres(sequelize) ? 'COALESCE' : 'IFNULL'
}
function jsonArrayContainsAny(columnExpression, bindName, sequelize) {
if (isPostgres(sequelize)) {
return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:${bindName}))`
}
return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value IN (:${bindName}))`
}
function jsonArrayContainsValue(columnExpression, bindName, sequelize) {
if (isPostgres(sequelize)) {
return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :${bindName})`
}
return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value = :${bindName})`
}
function jsonArrayExpand(columnExpression, sequelize, options = {}) {
const alias = options.alias || 'json_each'
const textValues = options.textValues !== false
if (isPostgres(sequelize)) {
const fn = textValues ? 'jsonb_array_elements_text' : 'jsonb_array_elements'
return `${fn}(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS ${alias}(value)`
}
return `json_each(${columnExpression})`
}
function jsonPathText(columnExpression, pathSegments, sequelize) {
const path = Array.isArray(pathSegments) ? pathSegments : [pathSegments]
if (isPostgres(sequelize)) {
return `${columnExpression}::jsonb #>> '{${path.join(',')}}'`
}
return `json_extract(${columnExpression}, '$.${path.join('.')}')`
}
function jsonPathNumber(columnExpression, pathSegments, sequelize) {
if (isPostgres(sequelize)) {
return `NULLIF(${jsonPathText(columnExpression, pathSegments, sequelize)}, '')::double precision`
}
return `json_extract(${columnExpression}, '$.${[].concat(pathSegments).join('.')}')`
}
function safeTextToDoubleExpression(columnExpression, sequelize) {
if (isPostgres(sequelize)) {
return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(${columnExpression})::double precision ELSE NULL END`
}
return `CAST(${columnExpression} AS FLOAT)`
}
function safeTextToIntegerExpression(columnExpression, sequelize) {
if (isPostgres(sequelize)) {
return `CASE WHEN BTRIM(${columnExpression}) ~ '^[+-]?\\d+$' THEN BTRIM(${columnExpression})::integer ELSE NULL END`
}
return `CAST(${columnExpression} AS INTEGER)`
}
module.exports = {
getDialect,
isPostgres,
booleanLiteral,
noCaseSortExpression,
coalesceFunctionName,
jsonArrayContainsAny,
jsonArrayContainsValue,
jsonArrayExpand,
jsonPathText,
jsonPathNumber,
safeTextToDoubleExpression,
safeTextToIntegerExpression
}

160
test/server/Database.test.js Normal file
View file

@ -0,0 +1,160 @@
const { expect } = require('chai')
const sinon = require('sinon')
const fs = require('../../server/libs/fsExtra')
const Database = require('../../server/Database')
describe('Database', () => {
let originalDialect
let originalSequelize
let originalEnv
beforeEach(() => {
originalDialect = Database.dialect
originalSequelize = Database.sequelize
originalEnv = {
DB_DIALECT: process.env.DB_DIALECT,
DATABASE_URL: process.env.DATABASE_URL
}
})
afterEach(() => {
Database.dialect = originalDialect
Database.sequelize = originalSequelize
if (originalEnv.DB_DIALECT === undefined) delete process.env.DB_DIALECT
else process.env.DB_DIALECT = originalEnv.DB_DIALECT
if (originalEnv.DATABASE_URL === undefined) delete process.env.DATABASE_URL
else process.env.DATABASE_URL = originalEnv.DATABASE_URL
sinon.restore()
})
describe('getConfiguredDialect', () => {
it('should default to sqlite when no env variables are set', () => {
delete process.env.DB_DIALECT
delete process.env.DATABASE_URL
expect(Database.getConfiguredDialect()).to.equal('sqlite')
})
it('should use explicit DB_DIALECT value when valid', () => {
process.env.DB_DIALECT = 'postgres'
process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite'
expect(Database.getConfiguredDialect()).to.equal('postgres')
})
it('should infer postgres dialect from DATABASE_URL', () => {
delete process.env.DB_DIALECT
process.env.DATABASE_URL = 'postgresql://user:pass@localhost:5432/abs'
expect(Database.getConfiguredDialect()).to.equal('postgres')
})
it('should fallback to sqlite for unsupported DB_DIALECT', () => {
process.env.DB_DIALECT = 'mysql'
process.env.DATABASE_URL = 'sqlite:///tmp/abs.sqlite'
expect(Database.getConfiguredDialect()).to.equal('sqlite')
})
})
describe('checkHasDb', () => {
it('should not check sqlite file existence in postgres mode', async () => {
Database.dialect = 'postgres'
const pathExistsStub = sinon.stub(fs, 'pathExists')
const hasDb = await Database.checkHasDb()
expect(hasDb).to.equal(true)
expect(pathExistsStub.called).to.equal(false)
})
})
describe('checkHasTables', () => {
it('should return true when at least one table exists', async () => {
Database.sequelize = {
getQueryInterface: () => ({
showAllTables: sinon.stub().resolves(['users'])
})
}
const hasTables = await Database.checkHasTables()
expect(hasTables).to.equal(true)
})
it('should return false when no tables exist', async () => {
Database.sequelize = {
getQueryInterface: () => ({
showAllTables: sinon.stub().resolves([])
})
}
const hasTables = await Database.checkHasTables()
expect(hasTables).to.equal(false)
})
})
describe('addPostgresTriggers', () => {
function captureQueries(existingTriggers = []) {
const queries = []
Database.sequelize = {
query: async (sql) => {
queries.push(sql)
const count = existingTriggers.filter((name) => sql.includes(`tgname = '${name}'`)).length
return [[{ count }]]
}
}
return queries
}
it('should create title and author names triggers with folded lowercase identifiers', async () => {
const queries = captureQueries()
await Database.addPostgresTriggers()
const functions = queries.filter((sql) => sql.includes('CREATE OR REPLACE FUNCTION'))
const triggers = queries.filter((sql) => sql.includes('CREATE TRIGGER'))
expect(functions.length).to.equal(7)
expect(triggers.length).to.equal(7)
const allDdl = [...functions, ...triggers].join('\n')
// No camelCase identifiers may leak into postgres DDL - unquoted identifiers fold to lowercase
expect(allDdl).to.not.match(/libraryItems|bookAuthors|mediaId|titleIgnorePrefix|authorNames|bookId|authorId|lastFirst|createdAt/)
const authorTrigger = triggers.find((sql) => sql.includes('update_library_items_author_names_on_authors_update'))
expect(authorTrigger).to.include('AFTER UPDATE OF name ON authors')
const insertFn = functions.find((sql) => sql.includes('update_library_items_author_names_on_book_authors_insert_fn'))
expect(insertFn).to.include("string_agg(authors.name, ', ' ORDER BY bookauthors.createdat ASC)")
expect(insertFn).to.include('WHERE mediaid = NEW.bookid')
const deleteFn = functions.find((sql) => sql.includes('update_library_items_author_names_on_book_authors_delete_fn'))
expect(deleteFn).to.include('WHERE mediaid = OLD.bookid')
})
it('should skip triggers that already exist', async () => {
const queries = captureQueries(['update_library_items_title'])
await Database.addPostgresTriggers()
const titleFn = queries.find((sql) => sql.includes('update_library_items_title_fn'))
expect(titleFn).to.equal(undefined)
const otherFns = queries.filter((sql) => sql.includes('CREATE OR REPLACE FUNCTION'))
expect(otherFns.length).to.equal(6)
})
it('should dispatch to postgres triggers from addTriggers', async () => {
Database.dialect = 'postgres'
const queries = captureQueries()
await Database.addTriggers()
expect(queries.some((sql) => sql.includes('CREATE TRIGGER'))).to.equal(true)
})
})
})

189
test/server/managers/BackupManager.test.js Normal file
View file

@ -0,0 +1,189 @@
const { expect } = require('chai')
const sinon = require('sinon')
const os = require('os')
const Path = require('path')
const EventEmitter = require('events')
const childProcess = require('child_process')
const sqlite3 = require('sqlite3')
const BackupManager = require('../../../server/managers/BackupManager')
const Backup = require('../../../server/objects/Backup')
const Database = require('../../../server/Database')
describe('BackupManager', () => {
let originalDialect
let originalDbPath
let originalConfigPath
let originalMetadataPath
beforeEach(() => {
originalDialect = Database.dialect
originalDbPath = Database.dbPath
originalConfigPath = global.ConfigPath
originalMetadataPath = global.MetadataPath
global.MetadataPath = os.tmpdir()
})
afterEach(() => {
Database.dialect = originalDialect
Database.dbPath = originalDbPath
global.ConfigPath = originalConfigPath
global.MetadataPath = originalMetadataPath
sinon.restore()
})
it('should select Postgres custom-format backups for the Postgres dialect', () => {
Database.dialect = 'postgres'
const manager = new BackupManager()
expect(manager.databaseBackupConfig).to.deep.equal({
dialect: 'postgres',
entryName: 'absdatabase.postgres.dump'
})
})
it('should create Postgres dumps with pg_dump without exposing credentials in argv', async () => {
Database.dialect = 'postgres'
Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf'
global.ConfigPath = os.tmpdir()
const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => {
callback(null, '', '')
})
const manager = new BackupManager()
const backup = new Backup()
backup.id = '2026-08-02T0130'
const dumpPath = await manager.backupPostgresDb(backup)
expect(dumpPath).to.equal(Path.join(os.tmpdir(), 'absdatabase.2026-08-02T0130.postgres.dump'))
expect(execFileStub.calledOnce).to.equal(true)
expect(execFileStub.firstCall.args[0]).to.equal('pg_dump')
expect(execFileStub.firstCall.args[1]).to.deep.equal([
'--format=custom',
'--no-owner',
'--no-acl',
'--file',
dumpPath,
'--host',
'localhost',
'--dbname',
'audiobookshelf',
'--port',
'5432',
'--username',
'absuser'
])
expect(execFileStub.firstCall.args[1].join(' ')).to.not.include('secretpass')
expect(execFileStub.firstCall.args[2].env.PGPASSWORD).to.equal('secretpass')
expect(execFileStub.firstCall.args[2].timeout).to.be.a('number')
})
it('should restore Postgres dumps in one transaction and clean existing objects', async () => {
Database.dialect = 'postgres'
Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf'
const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => {
callback(null, '', '')
})
const manager = new BackupManager()
await manager.restorePostgresDb('/config/absdatabase-postgres-temp.dump')
expect(execFileStub.firstCall.args[0]).to.equal('pg_restore')
expect(execFileStub.firstCall.args[1]).to.deep.equal([
'--clean',
'--if-exists',
'--exit-on-error',
'--single-transaction',
'--no-owner',
'--no-acl',
'/config/absdatabase-postgres-temp.dump',
'--host',
'localhost',
'--dbname',
'audiobookshelf',
'--port',
'5432',
'--username',
'absuser'
])
expect(execFileStub.firstCall.args[1].join(' ')).to.not.include('secretpass')
expect(execFileStub.firstCall.args[2].env.PGPASSWORD).to.equal('secretpass')
})
it('should redact database credentials from failed pg command errors', async () => {
Database.dialect = 'postgres'
Database.dbPath = 'postgresql://absuser:secretpass@localhost:5432/audiobookshelf'
global.ConfigPath = os.tmpdir()
sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => {
const error = new Error('Command failed: pg_dump --dbname postgresql://absuser:secretpass@localhost/audiobookshelf\npg_dump: error: password authentication failed')
error.cmd = 'pg_dump --dbname postgresql://absuser:secretpass@localhost/audiobookshelf'
callback(error, '', 'connection using password secretpass failed')
})
const manager = new BackupManager()
const backup = new Backup()
backup.id = '2026-08-02T0130'
let error
try {
await manager.backupPostgresDb(backup)
} catch (caughtError) {
error = caughtError
}
expect(error).to.be.an('error')
expect(error.message).to.not.include('secretpass')
expect(error.cmd).to.not.include('secretpass')
expect(error.stderr).to.not.include('secretpass')
expect(error.message).to.include('***')
})
it('should reject pg commands when DATABASE_URL is not a valid URI', async () => {
Database.dialect = 'postgres'
Database.dbPath = 'not a connection uri'
global.ConfigPath = os.tmpdir()
const manager = new BackupManager()
const backup = new Backup()
backup.id = '2026-08-02T0130'
let error
try {
await manager.backupPostgresDb(backup)
} catch (caughtError) {
error = caughtError
}
expect(error).to.be.an('error')
expect(error.message).to.include('valid postgres connection URI')
})
it('should reject SQLite backup open errors without an uncaught sqlite event', async () => {
Database.dialect = 'sqlite'
Database.dbPath = '/config/absdatabase.sqlite'
global.ConfigPath = os.tmpdir()
sinon.stub(sqlite3, 'Database').callsFake(function (_dbPath, callback) {
const db = new EventEmitter()
db.close = (closeCallback) => closeCallback()
process.nextTick(() => callback(Object.assign(new Error('unable to open database file'), { code: 'SQLITE_CANTOPEN' })))
return db
})
const manager = new BackupManager()
const backup = new Backup()
backup.id = '2026-08-02T0130'
let error
try {
await manager.backupSqliteDb(backup)
} catch (caughtError) {
error = caughtError
}
expect(error).to.be.an('error')
expect(error.code).to.equal('SQLITE_CANTOPEN')
})
})

60
test/server/managers/MigrationManager.test.js
View file

@ -189,6 +189,52 @@ describe('MigrationManager', () => {
expect(loggerInfoStub.calledWith(sinon.match('Restored the original database'))).to.be.true
expect(processExitStub.calledOnce).to.be.true
})
it('should skip sqlite backup workflow for postgres migrations', async () => {
// Arrange
migrationManager.serverVersion = '1.2.0'
migrationManager.databaseVersion = '1.1.0'
migrationManager.maxVersion = '1.1.0'
migrationManager.initialized = true
sequelizeStub.getDialect.returns('postgres')
umzugStub.migrations.resolves([{ name: 'v1.2.0-migration.js' }])
umzugStub.executed.resolves([{ name: 'v1.1.0-migration.js' }])
// Act
await migrationManager.runMigrations()
// Assert
expect(umzugStub.up.calledOnce).to.be.true
expect(fsCopyStub.called).to.be.false
expect(fsRemoveStub.called).to.be.false
})
})
describe('tableExists', () => {
it('should use queryInterface.tableExists when available', async () => {
const tableExistsStub = sinon.stub().resolves(true)
const sequelize = sinon.createStubInstance(Sequelize)
sequelize.getQueryInterface.returns({ tableExists: tableExistsStub })
const manager = new MigrationManager(sequelize, false, configPath)
const exists = await manager.tableExists('migrationsMeta')
expect(exists).to.equal(true)
expect(tableExistsStub.calledOnceWithExactly('migrationsMeta')).to.equal(true)
})
it('should fallback to showAllTables and support object table names', async () => {
const sequelize = sinon.createStubInstance(Sequelize)
sequelize.getQueryInterface.returns({
showAllTables: sinon.stub().resolves([{ tableName: 'migrationsMeta' }])
})
const manager = new MigrationManager(sequelize, false, configPath)
const exists = await manager.tableExists('migrationsMeta')
expect(exists).to.equal(true)
})
})
describe('fetchVersionsFromDatabase', () => {
@ -280,6 +326,20 @@ describe('MigrationManager', () => {
expect(error.message).to.equal('Database query failed')
}
})
it('should support lowercase maxversion alias from postgres drivers', async () => {
const sequelize = sinon.createStubInstance(Sequelize)
sequelize.query.onFirstCall().resolves([{ version: '1.1.0' }])
sequelize.query.onSecondCall().resolves([{ maxversion: '1.2.0' }])
const manager = new MigrationManager(sequelize, false, configPath)
manager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves()
await manager.fetchVersionsFromDatabase()
expect(manager.databaseVersion).to.equal('1.1.0')
expect(manager.maxVersion).to.equal('1.2.0')
})
})
describe('updateMaxVersion', () => {

17
test/server/migrations/v2.15.0-series-column-unique.test.js
View file

@ -88,6 +88,23 @@ describe('migration-v2.15.0-series-column-unique', () => {
await queryInterface.dropTable('Series')
await queryInterface.dropTable('BookSeries')
})
it('should skip NOCASE reindex on postgres dialect', async () => {
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
const originalQuery = queryInterface.sequelize.query.bind(queryInterface.sequelize)
const queryStub = sinon.stub(queryInterface.sequelize, 'query').callsFake((sql, options) => {
if (sql === 'REINDEX NOCASE;') {
throw new Error('Unexpected NOCASE reindex in postgres mode')
}
return originalQuery(sql, options)
})
await up({ context: { queryInterface, logger: Logger } })
expect(queryStub.neverCalledWith('REINDEX NOCASE;')).to.be.true
expect(loggerInfoStub.calledWith(sinon.match('[2.15.0 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.be.true
})
it('upgrade with no duplicate series', async () => {
// Add some entries to the Series table using the UUID for the ids
await queryInterface.bulkInsert('Series', [

41
test/server/migrations/v2.15.1-reindex-nocase.test.js Normal file
View file

@ -0,0 +1,41 @@
const { expect } = require('chai')
const sinon = require('sinon')
const { up, down } = require('../../../server/migrations/v2.15.1-reindex-nocase')
const Logger = require('../../../server/Logger')
describe('migration-v2.15.1-reindex-nocase', () => {
afterEach(() => {
sinon.restore()
})
it('should skip reindex on non-sqlite dialect', async () => {
const queryStub = sinon.stub().resolves()
const loggerInfoStub = sinon.stub(Logger, 'info')
const queryInterface = {
sequelize: {
getDialect: () => 'postgres',
query: queryStub
}
}
await up({ context: { queryInterface, logger: Logger } })
expect(queryStub.called).to.equal(false)
expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] Skipping NOCASE reindex on non-sqlite dialect'))).to.equal(true)
})
it('should log no-op on down migration', async () => {
const loggerInfoStub = sinon.stub(Logger, 'info')
const queryInterface = {
sequelize: {
getDialect: () => 'postgres',
query: sinon.stub().resolves()
}
}
await down({ context: { queryInterface, logger: Logger } })
expect(loggerInfoStub.calledWith(sinon.match('[2.15.1 migration] No action required for downgrade'))).to.equal(true)
})
})

98
test/server/migrations/v2.15.2-index-creation.test.js Normal file
View file

@ -0,0 +1,98 @@
const { expect } = require('chai')
const { Sequelize, DataTypes } = require('sequelize')
const sinon = require('sinon')
const Logger = require('../../../server/Logger')
const { up, down } = require('../../../server/migrations/v2.15.2-index-creation')
describe('migration-v2.15.2-index-creation', () => {
let sequelize
let queryInterface
beforeEach(async () => {
sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
queryInterface = sequelize.getQueryInterface()
await queryInterface.createTable('bookAuthors', {
id: { type: DataTypes.INTEGER, primaryKey: true },
authorId: { type: DataTypes.INTEGER }
})
await queryInterface.createTable('bookSeries', {
id: { type: DataTypes.INTEGER, primaryKey: true },
seriesId: { type: DataTypes.INTEGER }
})
await queryInterface.createTable('podcastEpisodes', {
id: { type: DataTypes.INTEGER, primaryKey: true },
createdAt: { type: DataTypes.DATE },
podcastId: { type: DataTypes.INTEGER }
})
})
afterEach(async () => {
if (sequelize) await sequelize.close()
})
it('up should succeed when legacy podcast index is missing', async () => {
await up({ context: { queryInterface, logger: Logger } })
const indexes = await queryInterface.showIndex('podcastEpisodes')
expect(indexes.some((index) => index.name === 'podcastEpisode_createdAt_podcastId')).to.equal(true)
})
it('down should succeed when new podcast index is missing', async () => {
await down({ context: { queryInterface, logger: Logger } })
const indexes = await queryInterface.showIndex('podcastEpisodes')
expect(indexes.some((index) => index.name === 'podcast_episodes_created_at')).to.equal(true)
})
it('up should treat index names case-insensitively', async () => {
const qi = {
showIndex: sinon.stub(),
addIndex: sinon.stub().resolves(),
removeIndex: sinon.stub().resolves()
}
qi.showIndex.onCall(0).resolves([{ name: 'bookauthor_authorid' }])
qi.showIndex.onCall(1).resolves([{ name: 'bookseries_seriesid' }])
qi.showIndex.onCall(2).resolves([{ name: 'podcast_episodes_created_at' }])
qi.showIndex.onCall(3).resolves([{ name: 'podcastepisode_createdat_podcastid' }])
await up({ context: { queryInterface: qi, logger: Logger } })
expect(qi.addIndex.called).to.equal(false)
expect(qi.removeIndex.calledOnceWithExactly('podcastEpisodes', 'podcast_episodes_created_at')).to.equal(true)
})
it('up should continue when addIndex reports already exists', async () => {
const makePgExistsError = (sql) => {
const err = new Error('relation already exists')
err.name = 'SequelizeDatabaseError'
err.original = { code: '42P07' }
err.sql = sql
return err
}
const qi = {
showIndex: sinon.stub(),
addIndex: sinon.stub(),
removeIndex: sinon.stub().resolves()
}
qi.showIndex.onCall(0).resolves([])
qi.showIndex.onCall(1).resolves([])
qi.showIndex.onCall(2).resolves([{ name: 'podcast_episodes_created_at' }])
qi.showIndex.onCall(3).resolves([])
qi.addIndex.onCall(0).rejects(makePgExistsError('CREATE INDEX bookAuthor_authorId ON bookAuthors (authorId)'))
qi.addIndex.onCall(1).rejects(makePgExistsError('CREATE INDEX bookSeries_seriesId ON bookSeries (seriesId)'))
qi.addIndex.onCall(2).rejects(makePgExistsError('CREATE INDEX podcastEpisode_createdAt_podcastId ON podcastEpisodes (createdAt, podcastId)'))
await up({ context: { queryInterface: qi, logger: Logger } })
expect(qi.addIndex.callCount).to.equal(3)
expect(qi.removeIndex.calledOnceWithExactly('podcastEpisodes', 'podcast_episodes_created_at')).to.equal(true)
})
})

10
test/server/migrations/v2.17.3-fk-constraints.test.js
View file

@ -34,6 +34,16 @@ describe('migration-v2.17.3-fk-constraints', () => {
await queryInterface.dropAllTables()
})
it('should skip sqlite-specific migration on non-sqlite dialect', async () => {
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
const queryStub = sinon.stub(queryInterface.sequelize, 'query').resolves([])
await up({ context: { queryInterface, logger: Logger } })
expect(queryStub.called).to.equal(false)
expect(loggerInfoStub.calledWith(sinon.match('[2.17.3 migration] Skipping sqlite-specific foreign key rewrite on non-sqlite dialect'))).to.equal(true)
})
it('should fix table foreign key constraints', async () => {
// Create tables with missing foreign key constraints: libraryItems, feeds, mediaItemShares, playbackSessions, playlistMediaItems, mediaProgresses
await queryInterface.sequelize.query('CREATE TABLE `libraryItems` (`id` UUID UNIQUE PRIMARY KEY, `libraryId` UUID REFERENCES `libraries` (`id`), `libraryFolderId` UUID REFERENCES `libraryFolders` (`id`));')

63
test/server/migrations/v2.17.4-use-subfolder-for-oidc-redirect-uris.test.js
View file

@ -30,11 +30,16 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
expect(logger.info.calledWith('[2.17.4 migration] UPGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] OIDC is enabled, adding authOpenIDSubfolderForRedirectURLs to server settings')).to.be.true
expect(queryInterface.sequelize.query.calledTwice).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true
expect(
queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = "server-settings";', {
queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(
queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', {
replacements: {
value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' })
value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }),
settingsKey: 'server-settings'
}
})
).to.be.true
@ -49,10 +54,31 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
expect(logger.info.calledWith('[2.17.4 migration] UPGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] OIDC is not enabled, no action required')).to.be.true
expect(queryInterface.sequelize.query.calledOnce).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true
expect(
queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] UPGRADE END: 2.17.4-use-subfolder-for-oidc-redirect-uris')).to.be.true
})
it('should handle already-parsed object server settings', async () => {
queryInterface.sequelize.query.onFirstCall().resolves([[{ value: { authActiveAuthMethods: ['openid'] } }]])
queryInterface.sequelize.query.onSecondCall().resolves()
await up({ context })
expect(queryInterface.sequelize.query.calledTwice).to.be.true
expect(
queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', {
replacements: {
value: JSON.stringify({ authActiveAuthMethods: ['openid'], authOpenIDSubfolderForRedirectURLs: '' }),
settingsKey: 'server-settings'
}
})
).to.be.true
})
it('should throw an error if server settings cannot be parsed', async () => {
queryInterface.sequelize.query.onFirstCall().resolves([[{ value: 'invalid json' }]])
@ -60,7 +86,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
await up({ context })
} catch (error) {
expect(queryInterface.sequelize.query.calledOnce).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true
expect(
queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(logger.error.calledWith('[2.17.4 migration] Error parsing server settings:')).to.be.true
expect(error).to.be.instanceOf(Error)
}
@ -73,7 +103,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
await up({ context })
} catch (error) {
expect(queryInterface.sequelize.query.calledOnce).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true
expect(
queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(logger.error.calledWith('[2.17.4 migration] Server settings not found')).to.be.true
expect(error).to.be.instanceOf(Error)
}
@ -90,11 +124,16 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] Removing authOpenIDSubfolderForRedirectURLs from server settings')).to.be.true
expect(queryInterface.sequelize.query.calledTwice).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true
expect(
queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = "server-settings";', {
queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(
queryInterface.sequelize.query.calledWith('UPDATE settings SET value = :value WHERE key = :settingsKey;', {
replacements: {
value: JSON.stringify({})
value: JSON.stringify({}),
settingsKey: 'server-settings'
}
})
).to.be.true
@ -109,7 +148,11 @@ describe('Migration v2.17.4-use-subfolder-for-oidc-redirect-uris', () => {
expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE BEGIN: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] authOpenIDSubfolderForRedirectURLs not found in server settings, no action required')).to.be.true
expect(queryInterface.sequelize.query.calledOnce).to.be.true
expect(queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = "server-settings";')).to.be.true
expect(
queryInterface.sequelize.query.calledWith('SELECT value FROM settings WHERE key = :settingsKey;', {
replacements: { settingsKey: 'server-settings' }
})
).to.be.true
expect(logger.info.calledWith('[2.17.4 migration] DOWNGRADE END: 2.17.4-use-subfolder-for-oidc-redirect-uris ')).to.be.true
})
})

12
test/server/migrations/v2.19.1-copy-title-to-library-items.test.js
View file

@ -47,6 +47,18 @@ describe('Migration v2.19.1-copy-title-to-library-items', () => {
})
describe('up', () => {
it('should skip sqlite-specific migration on non-sqlite dialect', async () => {
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
await up({ context: { queryInterface, logger: Logger } })
const table = await queryInterface.describeTable('libraryItems')
expect(table).to.have.property('id')
expect(table).to.not.have.property('title')
expect(table).to.not.have.property('titleIgnorePrefix')
expect(loggerInfoStub.calledWith(sinon.match('[2.19.1 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true
})
it('should copy title and titleIgnorePrefix to libraryItems', async () => {
await up({ context: { queryInterface, logger: Logger } })

12
test/server/migrations/v2.19.4-improve-podcast-queries.test.js
View file

@ -75,6 +75,18 @@ describe('Migration v2.19.4-improve-podcast-queries', () => {
})
describe('up', () => {
it('should skip sqlite-specific migration on non-sqlite dialect', async () => {
sinon.stub(queryInterface.sequelize, 'getDialect').returns('postgres')
await up({ context: { queryInterface, logger: Logger } })
const podcastsTable = await queryInterface.describeTable('podcasts')
const mediaProgressesTable = await queryInterface.describeTable('mediaProgresses')
expect(podcastsTable).to.not.have.property('numEpisodes')
expect(mediaProgressesTable).to.not.have.property('podcastId')
expect(loggerInfoStub.calledWith(sinon.match('[2.19.4 migration] skipping sqlite-specific migration on non-sqlite dialect'))).to.be.true
})
it('should add numEpisodes column to podcasts', async () => {
await up({ context: { queryInterface, logger: Logger } })

98
test/server/migrations/v2.35.0-add-last-refresh-token.test.js Normal file
View file

@ -0,0 +1,98 @@
const { expect } = require('chai')
const sinon = require('sinon')
const { up, down } = require('../../../server/migrations/v2.35.0-add-last-refresh-token')
const { Sequelize } = require('sequelize')
const Logger = require('../../../server/Logger')
describe('migration-v2.35.0-add-last-refresh-token', () => {
let sequelize
let queryInterface
let loggerInfoStub
beforeEach(() => {
sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
queryInterface = sequelize.getQueryInterface()
loggerInfoStub = sinon.stub(Logger, 'info')
})
afterEach(() => {
sinon.restore()
})
describe('up', () => {
beforeEach(async () => {
await queryInterface.createTable('sessions', {
id: { type: Sequelize.UUID, primaryKey: true },
createdAt: { type: Sequelize.DATE, allowNull: false },
updatedAt: { type: Sequelize.DATE, allowNull: false }
})
})
it('should add lastRefreshToken columns when they do not exist', async () => {
await up({ context: { queryInterface, logger: Logger } })
const tableDescription = await queryInterface.describeTable('sessions')
expect(tableDescription.lastRefreshToken).to.exist
expect(tableDescription.lastRefreshTokenExpiresAt).to.exist
})
it('should not fail when the columns already exist', async () => {
await up({ context: { queryInterface, logger: Logger } })
await up({ context: { queryInterface, logger: Logger } })
const tableDescription = await queryInterface.describeTable('sessions')
expect(tableDescription.lastRefreshToken).to.exist
expect(tableDescription.lastRefreshTokenExpiresAt).to.exist
expect(loggerInfoStub.calledWithMatch('lastRefreshToken column already exists')).to.equal(true)
})
it('should detect existing columns case-insensitively for postgres identifier folding', async () => {
// Postgres folds unquoted identifiers to lowercase, so describeTable on a
// migrated postgres database returns lowercase column names
const fakeQueryInterface = {
sequelize,
tableExists: async () => true,
describeTable: async () => ({
id: {},
lastrefreshtoken: {},
lastrefreshtokenexpiresat: {}
}),
addColumn: async () => {
throw new Error('addColumn must not be called for existing lowercase columns')
}
}
await up({ context: { queryInterface: fakeQueryInterface, logger: Logger } })
expect(loggerInfoStub.calledWithMatch('lastRefreshToken column already exists')).to.equal(true)
expect(loggerInfoStub.calledWithMatch('lastRefreshTokenExpiresAt column already exists')).to.equal(true)
})
})
describe('down', () => {
beforeEach(async () => {
await queryInterface.createTable('sessions', {
id: { type: Sequelize.UUID, primaryKey: true },
lastRefreshToken: { type: Sequelize.STRING, allowNull: true },
lastRefreshTokenExpiresAt: { type: Sequelize.DATE, allowNull: true },
createdAt: { type: Sequelize.DATE, allowNull: false },
updatedAt: { type: Sequelize.DATE, allowNull: false }
})
})
it('should remove lastRefreshToken columns when they exist', async () => {
await down({ context: { queryInterface, logger: Logger } })
const tableDescription = await queryInterface.describeTable('sessions')
expect(tableDescription.lastRefreshToken).to.not.exist
expect(tableDescription.lastRefreshTokenExpiresAt).to.not.exist
})
it('should not fail when the columns do not exist', async () => {
await down({ context: { queryInterface, logger: Logger } })
await down({ context: { queryInterface, logger: Logger } })
expect(loggerInfoStub.calledWithMatch('lastRefreshToken column does not exist')).to.equal(true)
})
})
})

70
test/server/models/MediaProgress.test.js Normal file
View file

@ -0,0 +1,70 @@
const { expect } = require('chai')
const sinon = require('sinon')
const Logger = require('../../../server/Logger')
const MediaProgress = require('../../../server/models/MediaProgress')
function createProgressInstance() {
const progress = Object.create(MediaProgress.prototype)
Object.defineProperties(progress, {
id: { value: 'progress-1', writable: true, configurable: true },
mediaItemId: { value: 'media-1', writable: true, configurable: true },
duration: { value: 3600, writable: true, configurable: true },
currentTime: { value: 120, writable: true, configurable: true },
isFinished: { value: false, writable: true, configurable: true },
hideFromContinueListening: { value: false, writable: true, configurable: true },
extraData: { value: {}, writable: true, configurable: true }
})
progress.changed = sinon.stub().returns(false)
progress.set = sinon.stub().callsFake((payload) => Object.assign(progress, payload))
progress.save = sinon.stub().resolves()
progress.reload = sinon.stub().resolves()
progress.constructor = {
update: sinon.stub().resolves(),
sequelize: {
escape: (value) => `'${value.toISOString ? value.toISOString() : value}'`
}
}
return progress
}
describe('MediaProgress', () => {
afterEach(() => {
sinon.restore()
})
describe('applyProgressUpdate', () => {
it('should update updatedAt via model update for valid lastUpdate', async () => {
const progress = createProgressInstance()
const infoStub = sinon.stub(Logger, 'info')
const lastUpdate = '2026-03-03T01:00:00.000Z'
await progress.applyProgressUpdate({ currentTime: 130, lastUpdate })
expect(progress.save.calledOnce).to.equal(true)
expect(progress.constructor.update.calledOnce).to.equal(true)
expect(progress.constructor.update.firstCall.args[0].updatedAt.toISOString()).to.equal(new Date(lastUpdate).toISOString())
expect(progress.constructor.update.firstCall.args[1]).to.deep.equal({
where: { id: 'progress-1' },
silent: true
})
expect(progress.reload.calledOnce).to.equal(true)
expect(infoStub.calledWithMatch('[MediaProgress] Manually setting updatedAt')).to.equal(true)
})
it('should skip manual updatedAt update when lastUpdate is invalid', async () => {
const progress = createProgressInstance()
const warnStub = sinon.stub(Logger, 'warn')
await progress.applyProgressUpdate({ currentTime: 130, lastUpdate: 'invalid-date' })
expect(progress.save.calledOnce).to.equal(true)
expect(progress.constructor.update.called).to.equal(false)
expect(progress.reload.called).to.equal(false)
expect(warnStub.calledWithMatch('[MediaProgress] Invalid date provided for lastUpdate')).to.equal(true)
})
})
})

170
test/server/models/User.test.js Normal file
View file

@ -0,0 +1,170 @@
const { expect } = require('chai')
const sinon = require('sinon')
const User = require('../../../server/models/User')
describe('User model', () => {
describe('case-insensitive lookup helpers', () => {
afterEach(() => {
sinon.restore()
})
it('should query usernames case-insensitively', async () => {
User.sequelize = {
models: {
mediaProgress: {}
}
}
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
await User.getUserByUsername('Madison')
expect(findOneStub.calledOnce).to.equal(true)
const options = findOneStub.firstCall.args[0]
expect(options.where.attribute.fn).to.equal('LOWER')
expect(options.where.attribute.args[0].col).to.equal('username')
expect(options.where.logic).to.equal('madison')
})
it('should query emails case-insensitively', async () => {
User.sequelize = {
models: {
mediaProgress: {}
}
}
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
await User.getUserByEmail('Example.User@Example.com')
expect(findOneStub.calledOnce).to.equal(true)
const options = findOneStub.firstCall.args[0]
expect(options.where.attribute.fn).to.equal('LOWER')
expect(options.where.attribute.args[0].col).to.equal('email')
expect(options.where.logic).to.equal('example.user@example.com')
})
it('should hit the user cache for mixed-case username lookups', async () => {
User.sequelize = {
models: {
mediaProgress: {}
}
}
const cachedUser = { id: 'cache-test-user-1', username: 'CacheTestUser', email: 'cachetest@example.com', extraData: {} }
const findOneStub = sinon.stub(User, 'findOne').resolves(cachedUser)
await User.getUserByUsername('CacheTestUser')
await User.getUserByUsername('cachetestuser')
await User.getUserByUsername('CACHETESTUSER')
expect(findOneStub.callCount).to.equal(1)
})
it('should hit the user cache for mixed-case email lookups', async () => {
User.sequelize = {
models: {
mediaProgress: {}
}
}
const cachedUser = { id: 'cache-test-user-2', username: 'CacheEmailUser', email: 'CacheMail@Example.com', extraData: {} }
const findOneStub = sinon.stub(User, 'findOne').resolves(cachedUser)
await User.getUserByEmail('CacheMail@Example.com')
await User.getUserByEmail('cachemail@example.com')
await User.getUserByEmail('CACHEMAIL@EXAMPLE.COM')
expect(findOneStub.callCount).to.equal(1)
})
})
describe('getUserByIdOrOldId', () => {
let originalSequelize
beforeEach(() => {
originalSequelize = User.sequelize
})
afterEach(() => {
User.sequelize = originalSequelize
sinon.restore()
})
it('should resolve UUID ids via primary-key lookup on postgres', async () => {
User.sequelize = {
getDialect: () => 'postgres',
models: {
mediaProgress: {}
}
}
const user = { id: 'e8e677b2-da16-4220-ab67-443b7714caf9' }
const findByPkStub = sinon.stub(User, 'findByPk').resolves(user)
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
const result = await User.getUserByIdOrOldId('e8e677b2-da16-4220-ab67-443b7714caf9')
expect(result).to.equal(user)
expect(findByPkStub.calledOnce).to.equal(true)
expect(findOneStub.called).to.equal(false)
})
it('should resolve uppercase UUID ids via primary-key lookup on postgres', async () => {
User.sequelize = {
getDialect: () => 'postgres',
models: {
mediaProgress: {}
}
}
const uppercaseUuid = 'E8E677B2-DA16-4220-AB67-443B7714CAF9'
const user = { id: uppercaseUuid }
const findByPkStub = sinon.stub(User, 'findByPk').resolves(user)
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
const result = await User.getUserByIdOrOldId(uppercaseUuid)
expect(result).to.equal(user)
expect(findByPkStub.calledOnceWithExactly(uppercaseUuid, { include: User.sequelize.models.mediaProgress })).to.equal(true)
expect(findOneStub.called).to.equal(false)
})
it('should query legacy oldUserId with postgres-safe JSON matcher', async () => {
User.sequelize = {
getDialect: () => 'postgres',
models: {
mediaProgress: {}
}
}
const findByPkStub = sinon.stub(User, 'findByPk').resolves(null)
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
await User.getUserByIdOrOldId('root')
expect(findByPkStub.called).to.equal(false)
expect(findOneStub.calledOnce).to.equal(true)
const options = findOneStub.firstCall.args[0]
expect(options.where.attribute.val).to.equal("extradata#>>'{oldUserId}'")
expect(options.where.logic).to.equal('root')
})
it('should keep sqlite oldUserId matcher unchanged', async () => {
User.sequelize = {
getDialect: () => 'sqlite',
models: {
mediaProgress: {}
}
}
const findOneStub = sinon.stub(User, 'findOne').resolves(null)
await User.getUserByIdOrOldId('root')
expect(findOneStub.calledOnce).to.equal(true)
const options = findOneStub.firstCall.args[0]
expect(options.where).to.deep.equal({ 'extraData.oldUserId': 'root' })
})
})
})

149
test/server/scripts/migrateSqliteToPostgres.test.js Normal file
View file

@ -0,0 +1,149 @@
const { expect } = require('chai')
const sqlite3 = require('sqlite3')
const {
normalizeJson,
isIntegerCompatible,
convertValue,
findOverlongVarcharValues,
findIntegerTypeIssues
} = require('../../../server/scripts/migrateSqliteToPostgres')
function openMemoryDb() {
return new Promise((resolve, reject) => {
const db = new sqlite3.Database(':memory:', (error) => {
if (error) return reject(error)
resolve(db)
})
})
}
function run(db, sql, params = []) {
return new Promise((resolve, reject) => {
db.run(sql, params, (error) => {
if (error) return reject(error)
resolve()
})
})
}
function close(db) {
return new Promise((resolve, reject) => {
db.close((error) => {
if (error) return reject(error)
resolve()
})
})
}
describe('migrateSqliteToPostgres script helpers', () => {
it('should keep malformed JSON payloads insertable by returning valid JSON text', () => {
const malformed = '"{"x",1}"'
const normalized = normalizeJson(malformed)
expect(() => JSON.parse(normalized)).to.not.throw()
expect(JSON.parse(normalized)).to.equal('"{"x",1}"')
})
it('should detect overlong varchar values before migration', async () => {
const db = await openMemoryDb()
try {
await run(db, 'CREATE TABLE books (subtitle TEXT)')
await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['x'.repeat(300)])
await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['ok'])
const tablesToMigrate = [{ sqliteTable: 'books', postgresTable: 'books' }]
const pgColumnsByTable = new Map([
[
'books',
new Map([
[
'subtitle',
{
column_name: 'subtitle',
data_type: 'character varying',
character_maximum_length: 255
}
]
])
]
])
const issues = await findOverlongVarcharValues(db, tablesToMigrate, pgColumnsByTable)
expect(issues).to.have.length(1)
expect(issues[0]).to.include({
sqliteTable: 'books',
sqliteColumn: 'subtitle',
postgresTable: 'books',
postgresColumn: 'subtitle',
maxLength: 255,
actualMaxLength: 300,
overCount: 1
})
} finally {
await close(db)
}
})
it('should detect non-integer values for integer target columns', async () => {
const db = await openMemoryDb()
try {
await run(db, 'CREATE TABLE playbackSessions (timeListening REAL)')
await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [25.802536999999997])
await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [42])
const tablesToMigrate = [{ sqliteTable: 'playbackSessions', postgresTable: 'playbackSessions' }]
const pgColumnsByTable = new Map([
[
'playbackSessions',
new Map([
[
'timelistening',
{
column_name: 'timeListening',
data_type: 'integer'
}
]
])
]
])
const issues = await findIntegerTypeIssues(db, tablesToMigrate, pgColumnsByTable)
expect(issues).to.have.length(1)
expect(issues[0].sqliteTable).to.equal('playbackSessions')
expect(issues[0].sqliteColumn).to.equal('timeListening')
expect(issues[0].postgresType).to.equal('integer')
expect(issues[0].badCount).to.equal(1)
} finally {
await close(db)
}
})
it('should only accept integer-compatible values for integer columns', () => {
expect(isIntegerCompatible(10)).to.equal(true)
expect(isIntegerCompatible('10')).to.equal(true)
expect(isIntegerCompatible(10.5)).to.equal(false)
expect(isIntegerCompatible('10.5')).to.equal(false)
expect(isIntegerCompatible('32768', 'smallint')).to.equal(false)
expect(isIntegerCompatible('9007199254740993', 'bigint')).to.equal(true)
})
it('should coerce integer-like strings for postgres integer columns', () => {
expect(convertValue('42', { data_type: 'integer' })).to.equal(42)
expect(convertValue('-7', { data_type: 'bigint' })).to.equal('-7')
expect(convertValue('9007199254740993', { data_type: 'bigint' })).to.equal('9007199254740993')
expect(convertValue('4.2', { data_type: 'integer' })).to.equal('4.2')
})
it('should always return valid json text for postgres json columns', () => {
const convertedObject = convertValue({ a: 1 }, { data_type: 'jsonb', udt_name: 'jsonb' })
const convertedMalformed = convertValue('"{"x",1}"', { data_type: 'jsonb', udt_name: 'jsonb' })
expect(JSON.parse(convertedObject)).to.deep.equal({ a: 1 })
expect(JSON.parse(convertedMalformed)).to.equal('"{"x",1}"')
})
})

96
test/server/utils/queries/libraryFilters.test.js Normal file
View file

@ -0,0 +1,96 @@
const { expect } = require('chai')
const sinon = require('sinon')
const Database = require('../../../../server/Database')
const Logger = require('../../../../server/Logger')
const libraryFilters = require('../../../../server/utils/queries/libraryFilters')
const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters')
const libraryItemsPodcastFilters = require('../../../../server/utils/queries/libraryItemsPodcastFilters')
describe('libraryFilters shelf resilience', () => {
afterEach(() => {
sinon.restore()
})
it('should return empty discover shelf when discover query fails', async () => {
sinon.stub(Database, 'isPostgresDialect').returns(true)
sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').rejects(new Error('discover failed'))
const errorStub = sinon.stub(Logger, 'error')
const result = await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10)
expect(result).to.deep.equal({ libraryItems: [], count: 0 })
expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load discover shelf for library "library-1"')).to.equal(true)
})
it('should return empty in-progress shelf when a query fails', async () => {
sinon.stub(Database, 'isPostgresDialect').returns(true)
sinon.stub(libraryItemsBookFilters, 'getFilteredLibraryItems').rejects(new Error('progress failed'))
const errorStub = sinon.stub(Logger, 'error')
const result = await libraryFilters.getMediaItemsInProgress({ isBook: true, id: 'library-1' }, { id: 'user-1' }, [], 10)
expect(result).to.deep.equal({ items: [], count: 0 })
expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load in-progress shelf for library "library-1"')).to.equal(true)
})
it('should return empty continue-series shelf when a query fails', async () => {
sinon.stub(Database, 'isPostgresDialect').returns(true)
sinon.stub(libraryItemsBookFilters, 'getContinueSeriesLibraryItems').rejects(new Error('continue failed'))
const errorStub = sinon.stub(Logger, 'error')
const result = await libraryFilters.getLibraryItemsContinueSeries({ id: 'library-1' }, { id: 'user-1' }, [], 10)
expect(result).to.deep.equal({ libraryItems: [], count: 0 })
expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load continue-series shelf for library "library-1"')).to.equal(true)
})
it('should return empty newest podcast episodes shelf when a query fails', async () => {
sinon.stub(Database, 'isPostgresDialect').returns(true)
sinon.stub(libraryItemsPodcastFilters, 'getFilteredPodcastEpisodes').rejects(new Error('podcast failed'))
const errorStub = sinon.stub(Logger, 'error')
const result = await libraryFilters.getNewestPodcastEpisodes({ mediaType: 'podcast', id: 'library-1' }, { id: 'user-1' }, 10)
expect(result).to.deep.equal({ libraryItems: [], count: 0 })
expect(errorStub.calledWithMatch('[LibraryFilters] Failed to load newest-podcast-episodes shelf for library "library-1"')).to.equal(true)
})
it('should rethrow shelf query errors on sqlite to keep upstream behavior', async () => {
sinon.stub(Database, 'isPostgresDialect').returns(false)
sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').rejects(new Error('discover failed'))
let error
try {
await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10)
} catch (caughtError) {
error = caughtError
}
expect(error).to.be.an('error')
expect(error.message).to.equal('discover failed')
})
it('should keep discover shelf mapping behavior when query succeeds', async () => {
const libraryItem = {
toOldJSONMinified: () => ({ id: 'item-1' }),
rssFeed: { toOldJSONMinified: () => ({ id: 'rss-1' }) },
mediaItemShare: { id: 'share-1' }
}
sinon.stub(libraryItemsBookFilters, 'getDiscoverLibraryItems').resolves({
libraryItems: [libraryItem],
count: 1
})
const result = await libraryFilters.getLibraryItemsToDiscover({ mediaType: 'book', id: 'library-1' }, { id: 'user-1' }, [], 10)
expect(result.count).to.equal(1)
expect(result.libraryItems).to.deep.equal([
{
id: 'item-1',
rssFeed: { id: 'rss-1' },
mediaItemShare: { id: 'share-1' }
}
])
})
})

148
test/server/utils/queries/libraryItemsBookFilters.test.js Normal file
View file

@ -0,0 +1,148 @@
const { expect } = require('chai')
const sinon = require('sinon')
const Sequelize = require('sequelize')
const Database = require('../../../../server/Database')
const Logger = require('../../../../server/Logger')
const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters')
function createSequelizeStub(dialect, models) {
return {
getDialect: () => dialect,
escape: (value) => `'${String(value).replace(/'/g, "''")}'`,
random: () => ({ fn: 'random' }),
models
}
}
function createModelStubs() {
return {
book: {
findAndCountAll: sinon.stub(),
findAll: sinon.stub(),
count: sinon.stub()
},
libraryItem: {},
feed: {},
bookSeries: {},
series: {
findAll: sinon.stub()
},
bookAuthor: {},
author: {},
mediaProgress: {},
mediaItemShare: {}
}
}
describe('libraryItemsBookFilters postgres query safety', () => {
let originalSequelize
let originalServerSettings
let modelStubs
beforeEach(() => {
originalSequelize = Database.sequelize
originalServerSettings = global.ServerSettings
modelStubs = createModelStubs()
global.ServerSettings = {
sortingIgnorePrefix: false
}
})
afterEach(() => {
Database.sequelize = originalSequelize
global.ServerSettings = originalServerSettings
sinon.restore()
})
it('should avoid generating IN () when collapse-series has no include ids', async () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 })
sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({
booksToExclude: [],
bookSeriesToInclude: []
})
const debugStub = sinon.stub(Logger, 'debug')
await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, true, [], 20, 0)
const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0]
const displayTitleExpression = findOptions.attributes.include[0][0].val
expect(displayTitleExpression).to.include('COALESCE(NULL, libraryItem.title)')
expect(displayTitleExpression).to.not.include('IN ()')
expect(debugStub.calledWithMatch('collapse-series produced no include IDs')).to.equal(true)
})
it('should escape collapse-series ids safely for postgres subquery', async () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
modelStubs.book.findAndCountAll.resolves({ rows: [], count: 0 })
sinon.stub(libraryItemsBookFilters, 'getCollapseSeriesBooksToExclude').resolves({
booksToExclude: [],
bookSeriesToInclude: [{ id: "series-id-'quoted'", numBooks: 2, libraryItemIds: [] }]
})
global.ServerSettings.sortingIgnorePrefix = true
await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.title', false, true, [], 20, 0)
const findOptions = modelStubs.book.findAndCountAll.firstCall.args[0]
const displayTitleExpression = findOptions.attributes.include[0][0].val
expect(displayTitleExpression).to.include("bs.id IN ('series-id-''quoted''')")
expect(displayTitleExpression).to.include('libraryItem.titleIgnorePrefix')
})
it('should use postgres-safe join alias for sequence sorting', () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
const order = libraryItemsBookFilters.getOrder('sequence', false, false)
const expression = order[0][0].val
expect(expression).to.include('"series->bookSeries"."sequence"')
expect(expression).to.include('CASE WHEN BTRIM')
})
it('should log generated query when findAndCountAll fails', async () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
modelStubs.book.findAndCountAll.rejects(new Error('boom'))
const errorStub = sinon.stub(Logger, 'error')
try {
await libraryItemsBookFilters.getFilteredLibraryItems('library-1', { canAccessExplicitContent: true }, 'authors', 'author-1', 'media.metadata.publishedYear', true, false, [], 20, 0)
expect.fail('Expected getFilteredLibraryItems to throw')
} catch (error) {
expect(error.message).to.equal('boom')
}
expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll failed: boom')).to.equal(true)
expect(errorStub.calledWithMatch('[LibraryItemsBookFilters] findAndCountAll query:')).to.equal(true)
})
it('should use postgres-safe books->bookSeries alias in collapse-series selector query', async () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
modelStubs.series.findAll.resolves([])
await libraryItemsBookFilters.getCollapseSeriesBooksToExclude({ where: {}, include: [] }, null)
const findAllOptions = modelStubs.series.findAll.firstCall.args[0]
const orderExpression = findAllOptions.order[0].val
expect(orderExpression).to.include('"books->bookSeries"."sequence"')
})
it('should use boolean false for discover not-started media progress on postgres', async () => {
Database.sequelize = createSequelizeStub('postgres', modelStubs)
modelStubs.series.findAll.resolves([])
modelStubs.book.count.resolves(0)
modelStubs.book.findAll.resolves([])
await libraryItemsBookFilters.getDiscoverLibraryItems('library-1', { id: 'user-1', canAccessExplicitContent: true }, [], 10)
const countOptions = modelStubs.book.count.firstCall.args[0]
const progressFilter = countOptions.where[0]['$mediaProgresses.isFinished$']
expect(progressFilter[Sequelize.Op.or]).to.deep.equal([null, false])
})
})

75
test/server/utils/queries/libraryItemsPodcastFilters.test.js Normal file
View file

@ -0,0 +1,75 @@
const { expect } = require('chai')
const sinon = require('sinon')
const Database = require('../../../../server/Database')
const libraryItemsPodcastFilters = require('../../../../server/utils/queries/libraryItemsPodcastFilters')
describe('libraryItemsPodcastFilters dialect behavior', () => {
let originalSequelize
let originalServerSettings
beforeEach(() => {
originalSequelize = Database.sequelize
originalServerSettings = global.ServerSettings
global.ServerSettings = { sortingIgnorePrefix: false }
})
afterEach(() => {
Database.sequelize = originalSequelize
global.ServerSettings = originalServerSettings
sinon.restore()
})
it('should build postgres json tag permission predicate', () => {
Database.sequelize = { getDialect: () => 'postgres' }
const { podcastWhere, replacements } = libraryItemsPodcastFilters.getUserPermissionPodcastWhereQuery({
canAccessExplicitContent: true,
permissions: {
accessAllTags: false,
itemTagsSelected: ['fiction'],
selectedTagsNotAccessible: false
}
})
expect(replacements.userTagsSelected).to.deep.equal(['fiction'])
expect(podcastWhere[0].attribute.val).to.include('jsonb_array_elements_text')
})
it('should build sqlite no-case author sort expression', () => {
Database.sequelize = { getDialect: () => 'sqlite' }
const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false)
expect(order[0][0].val).to.include('podcast.author COLLATE NOCASE')
})
it('should build postgres lower author sort expression', () => {
Database.sequelize = { getDialect: () => 'postgres' }
const order = libraryItemsPodcastFilters.getOrder('media.metadata.author', false)
expect(order[0][0].val).to.include('LOWER(podcast.author)')
})
it('should use postgres json duration extraction in podcast stats query', async () => {
const queryStub = sinon.stub()
queryStub.onFirstCall().resolves([[{ totalSize: 1000 }]])
queryStub.onSecondCall().resolves([[{ totalDuration: '12.3', totalItems: '4', numAudioFiles: '9' }]])
Database.sequelize = {
getDialect: () => 'postgres',
query: queryStub
}
const result = await libraryItemsPodcastFilters.getPodcastLibraryStats('library-1')
expect(queryStub.secondCall.args[0]).to.include('NULLIF(pe.audioFile::jsonb #>>')
expect(result).to.deep.equal({
totalSize: 1000,
totalDuration: '12.3',
numAudioFiles: '9',
totalItems: '4'
})
})
})

107
test/server/utils/queries/seriesFilters.test.js Normal file
View file

@ -0,0 +1,107 @@
const { expect } = require('chai')
const sinon = require('sinon')
const Database = require('../../../../server/Database')
const seriesFilters = require('../../../../server/utils/queries/seriesFilters')
const libraryItemsBookFilters = require('../../../../server/utils/queries/libraryItemsBookFilters')
function encodedFilter(group, value) {
return `${group}.${encodeURIComponent(Buffer.from(value).toString('base64'))}`
}
describe('seriesFilters dialect behavior', () => {
let originalSequelize
let originalServerSettings
let modelStubs
function setDialect(dialect) {
Database.sequelize = {
getDialect: () => dialect,
random: sinon.stub(),
models: modelStubs
}
}
beforeEach(() => {
originalSequelize = Database.sequelize
originalServerSettings = global.ServerSettings
modelStubs = {
series: {
findAndCountAll: sinon.stub().resolves({ rows: [], count: 0 })
},
bookSeries: {},
book: {},
libraryItem: {},
author: {},
feed: {}
}
global.ServerSettings = { sortingIgnorePrefix: false }
sinon.stub(libraryItemsBookFilters, 'getUserPermissionBookWhereQuery').returns({
bookWhere: [],
replacements: {}
})
})
afterEach(() => {
Database.sequelize = originalSequelize
global.ServerSettings = originalServerSettings
sinon.restore()
})
it('should build postgres progress filter with TRUE and FALSE literals', async () => {
setDialect('postgres')
await seriesFilters.getFilteredSeries(
{ id: 'library-1', settings: { hideSingleBookSeries: false } },
{ id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } },
encodedFilter('progress', 'not-started'),
'name',
false,
[],
10,
0
)
const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0]
const progressWhere = findOptions.where[1]
expect(progressWhere.attribute.val).to.include('mp.isFinished = TRUE')
})
it('should build sqlite no-case name sort expression', async () => {
setDialect('sqlite')
await seriesFilters.getFilteredSeries(
{ id: 'library-1', settings: { hideSingleBookSeries: false } },
{ id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } },
null,
'name',
false,
[],
10,
0
)
const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0]
expect(findOptions.order[0][0].val).to.equal('series.name COLLATE NOCASE')
})
it('should build postgres lower-case name sort expression', async () => {
setDialect('postgres')
await seriesFilters.getFilteredSeries(
{ id: 'library-1', settings: { hideSingleBookSeries: false } },
{ id: 'user-1', canAccessExplicitContent: true, permissions: { accessAllTags: true } },
null,
'name',
false,
[],
10,
0
)
const findOptions = modelStubs.series.findAndCountAll.firstCall.args[0]
expect(findOptions.order[0][0].val).to.equal('LOWER(series.name)')
})
})

47
test/server/utils/queries/sqlBooleanSafety.test.js Normal file
View file

@ -0,0 +1,47 @@
const { expect } = require('chai')
const fs = require('fs')
const path = require('path')
function getJsFiles(dirPath) {
const entries = fs.readdirSync(dirPath, { withFileTypes: true })
return entries.flatMap((entry) => {
const entryPath = path.join(dirPath, entry.name)
if (entry.isDirectory()) return getJsFiles(entryPath)
if (entry.isFile() && entry.name.endsWith('.js')) return [entryPath]
return []
})
}
describe('query boolean safety', () => {
it('should not use numeric literals for SQL boolean fields in query code', () => {
const queriesDir = path.resolve(__dirname, '../../../../server/utils/queries')
const files = getJsFiles(queriesDir)
const offenders = []
const forbiddenPatterns = [
{
regex: /\$mediaProgresses\.isFinished\$[\s\S]{0,120}\[null,\s*[01]\]/g,
message: 'numeric literal in mediaProgresses.isFinished filter'
},
{
regex: /\$books\.mediaProgresses\.isFinished\$[\s\S]{0,120}\[null,\s*[01]\]/g,
message: 'numeric literal in books.mediaProgresses.isFinished filter'
},
{
regex: /\b(?:mp|mediaProgresses|b)\.(?:isFinished|explicit|abridged)\s*=\s*[01]\b/g,
message: 'direct SQL boolean comparison against 0/1'
}
]
files.forEach((filePath) => {
const content = fs.readFileSync(filePath, 'utf8')
forbiddenPatterns.forEach(({ regex, message }) => {
const matches = [...content.matchAll(regex)]
matches.forEach((match) => {
offenders.push(`${path.relative(queriesDir, filePath)}: ${message}: ${match[0].replace(/\s+/g, ' ').trim()}`)
})
})
})
expect(offenders).to.deep.equal([])
})
})

68
test/server/utils/sqlDialectHelpers.test.js Normal file
View file

@ -0,0 +1,68 @@
const { expect } = require('chai')
const {
booleanLiteral,
noCaseSortExpression,
coalesceFunctionName,
jsonArrayContainsAny,
jsonArrayContainsValue,
jsonArrayExpand,
jsonPathText,
jsonPathNumber,
safeTextToDoubleExpression,
safeTextToIntegerExpression
} = require('../../../server/utils/sqlDialectHelpers')
const sqlite = {
getDialect: () => 'sqlite'
}
const postgres = {
getDialect: () => 'postgres'
}
describe('sqlDialectHelpers', () => {
it('should return sqlite and postgres boolean literals', () => {
expect(booleanLiteral(true, sqlite)).to.equal('1')
expect(booleanLiteral(false, sqlite)).to.equal('0')
expect(booleanLiteral(true, postgres)).to.equal('TRUE')
expect(booleanLiteral(false, postgres)).to.equal('FALSE')
})
it('should generate case-insensitive sort expressions', () => {
expect(noCaseSortExpression('name', sqlite)).to.equal('name COLLATE NOCASE')
expect(noCaseSortExpression('name', postgres)).to.equal('LOWER(name)')
})
it('should choose the correct null-coalescing function name', () => {
expect(coalesceFunctionName(sqlite)).to.equal('IFNULL')
expect(coalesceFunctionName(postgres)).to.equal('COALESCE')
})
it('should generate array membership count queries per dialect', () => {
expect(jsonArrayContainsAny('tags', 'selectedTags', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:selectedTags))')
expect(jsonArrayContainsAny('tags', 'selectedTags', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:selectedTags))")
expect(jsonArrayContainsValue('tags', 'tag', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value = :tag)')
expect(jsonArrayContainsValue('tags', 'tag', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :tag)")
})
it('should generate json array expansion and path extraction by dialect', () => {
expect(jsonArrayExpand('books.tags', sqlite)).to.equal('json_each(books.tags)')
expect(jsonArrayExpand('books.tags', postgres)).to.equal("jsonb_array_elements_text(COALESCE(books.tags::jsonb, '[]'::jsonb)) AS json_each(value)")
expect(jsonPathText('payload', ['metadata', 'filename'], sqlite)).to.equal("json_extract(payload, '$.metadata.filename')")
expect(jsonPathText('payload', ['metadata', 'filename'], postgres)).to.equal("payload::jsonb #>> '{metadata,filename}'")
expect(jsonPathNumber('payload', ['duration'], sqlite)).to.equal("json_extract(payload, '$.duration')")
expect(jsonPathNumber('payload', ['duration'], postgres)).to.equal("NULLIF(payload::jsonb #>> '{duration}', '')::double precision")
})
it('should generate safe numeric cast expressions for sequence-like text', () => {
expect(safeTextToDoubleExpression('sequence', sqlite)).to.equal('CAST(sequence AS FLOAT)')
expect(safeTextToDoubleExpression('"bookSeries"."sequence"', postgres)).to.equal("CASE WHEN BTRIM(\"bookSeries\".\"sequence\") ~ '^[+-]?(?:\\d+\\.?\\d*|\\.\\d+)$' THEN BTRIM(\"bookSeries\".\"sequence\")::double precision ELSE NULL END")
expect(safeTextToIntegerExpression('publishedYear', sqlite)).to.equal('CAST(publishedYear AS INTEGER)')
expect(safeTextToIntegerExpression('book.publishedYear', postgres)).to.equal("CASE WHEN BTRIM(book.publishedYear) ~ '^[+-]?\\d+$' THEN BTRIM(book.publishedYear)::integer ELSE NULL END")
})
})