mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2026-07-07 18:01:42 +00:00
add postgres dialect support for homelab migration
This commit is contained in:
Kevin Gatera
parent
fa5fa7b788
commit
453cd2587b
20 changed files with 851 additions and 159 deletions
171
package-lock.json
generated
171
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "audiobookshelf",
|
||||
"version": "2.32.1",
|
||||
"version": "2.32.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "audiobookshelf",
|
||||
"version": "2.32.1",
|
||||
"version": "2.32.2",
|
||||
"license": "GPL-3.0",
|
||||
"dependencies": {
|
||||
"axios": "^0.27.2",
|
||||
|
|
@ -23,6 +23,8 @@
|
|||
"p-throttle": "^4.1.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-jwt": "^4.0.1",
|
||||
"pg": "^8.19.0",
|
||||
"pg-hstore": "^2.3.4",
|
||||
"semver": "^7.6.3",
|
||||
"sequelize": "^6.35.2",
|
||||
"socket.io": "^4.5.4",
|
||||
|
|
@ -4138,10 +4140,106 @@
|
|||
"resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz",
|
||||
"integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg=="
|
||||
},
|
||||
"node_modules/pg": {
|
||||
"version": "8.19.0",
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-8.19.0.tgz",
|
||||
"integrity": "sha512-QIcLGi508BAHkQ3pJNptsFz5WQMlpGbuBGBaIaXsWK8mel2kQ/rThYI+DbgjUvZrIr7MiuEuc9LcChJoEZK1xQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"pg-connection-string": "^2.11.0",
|
||||
"pg-pool": "^3.12.0",
|
||||
"pg-protocol": "^1.12.0",
|
||||
"pg-types": "2.2.0",
|
||||
"pgpass": "1.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 16.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"pg-cloudflare": "^1.3.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"pg-native": ">=3.0.1"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"pg-native": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/pg-cloudflare": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz",
|
||||
"integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==",
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/pg-connection-string": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz",
|
||||
"integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg=="
|
||||
"version": "2.11.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz",
|
||||
"integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pg-hstore": {
|
||||
"version": "2.3.4",
|
||||
"resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz",
|
||||
"integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"underscore": "^1.13.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.8.x"
|
||||
}
|
||||
},
|
||||
"node_modules/pg-int8": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
|
||||
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pg-pool": {
|
||||
"version": "3.12.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.12.0.tgz",
|
||||
"integrity": "sha512-eIJ0DES8BLaziFHW7VgJEBPi5hg3Nyng5iKpYtj3wbcAUV9A1wLgWiY7ajf/f/oO1wfxt83phXPY8Emztg7ITg==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"pg": ">=8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pg-protocol": {
|
||||
"version": "1.12.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.12.0.tgz",
|
||||
"integrity": "sha512-uOANXNRACNdElMXJ0tPz6RBM0XQ61nONGAwlt8da5zs/iUOOCLBQOHSXnrC6fMsvtjxbOJrZZl5IScGv+7mpbg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pg-types": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
|
||||
"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"pg-int8": "1.0.1",
|
||||
"postgres-array": "~2.0.0",
|
||||
"postgres-bytea": "~1.0.0",
|
||||
"postgres-date": "~1.0.4",
|
||||
"postgres-interval": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/pgpass": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
|
||||
"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"split2": "^4.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.0.0",
|
||||
|
|
@ -4173,6 +4271,45 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-array": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
|
||||
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-bytea": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
|
||||
"integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-date": {
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
|
||||
"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-interval": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
|
||||
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"xtend": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/prebuild-install": {
|
||||
"version": "7.1.3",
|
||||
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
|
||||
|
|
@ -4946,6 +5083,15 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/split2": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
|
||||
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">= 10.x"
|
||||
}
|
||||
},
|
||||
"node_modules/sprintf-js": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
|
||||
|
|
@ -5265,6 +5411,12 @@
|
|||
"integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/underscore": {
|
||||
"version": "1.13.8",
|
||||
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz",
|
||||
"integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/unique-filename": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz",
|
||||
|
|
@ -5476,6 +5628,15 @@
|
|||
"node": ">=4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/xtend": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
|
||||
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/y18n": {
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
"docker": "docker buildx build --platform linux/amd64,linux/arm64 --push . -t advplyr/audiobookshelf",
|
||||
"docker-amd64-local": "docker buildx build --platform linux/amd64 --load . -t advplyr/audiobookshelf-amd64-local",
|
||||
"docker-arm64-local": "docker buildx build --platform linux/arm64 --load . -t advplyr/audiobookshelf-arm64-local",
|
||||
"migrate-sqlite-to-postgres": "node server/scripts/migrateSqliteToPostgres.js",
|
||||
"deploy-linux": "node deploy/linux",
|
||||
"test": "mocha",
|
||||
"coverage": "nyc mocha"
|
||||
|
|
@ -52,6 +53,8 @@
|
|||
"p-throttle": "^4.1.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-jwt": "^4.0.1",
|
||||
"pg": "^8.19.0",
|
||||
"pg-hstore": "^2.3.4",
|
||||
"semver": "^7.6.3",
|
||||
"sequelize": "^6.35.2",
|
||||
"socket.io": "^4.5.4",
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ class Database {
|
|||
constructor() {
|
||||
this.sequelize = null
|
||||
this.dbPath = null
|
||||
this.dialect = 'sqlite'
|
||||
this.isNew = false // New absdatabase.sqlite created
|
||||
this.hasRootUser = false // Used to show initialization page in web ui
|
||||
|
||||
|
|
@ -162,11 +163,37 @@ class Database {
|
|||
return this.models.device
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {'sqlite'|'postgres'}
|
||||
*/
|
||||
getConfiguredDialect() {
|
||||
const explicitDialect = process.env.DB_DIALECT?.trim()?.toLowerCase()
|
||||
if (explicitDialect === 'postgres' || explicitDialect === 'sqlite') {
|
||||
return explicitDialect
|
||||
}
|
||||
|
||||
const databaseUrl = process.env.DATABASE_URL || ''
|
||||
if (databaseUrl.startsWith('postgres://') || databaseUrl.startsWith('postgresql://')) {
|
||||
return 'postgres'
|
||||
}
|
||||
|
||||
return 'sqlite'
|
||||
}
|
||||
|
||||
isSqliteDialect() {
|
||||
return this.dialect === 'sqlite'
|
||||
}
|
||||
|
||||
isPostgresDialect() {
|
||||
return this.dialect === 'postgres'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if db file exists
|
||||
* @returns {boolean}
|
||||
*/
|
||||
async checkHasDb() {
|
||||
if (!this.isSqliteDialect()) return true
|
||||
if (!(await fs.pathExists(this.dbPath))) {
|
||||
Logger.info(`[Database] absdatabase.sqlite not found at ${this.dbPath}`)
|
||||
return false
|
||||
|
|
@ -174,20 +201,39 @@ class Database {
|
|||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if any user tables exist (for networked dialects)
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async checkHasTables() {
|
||||
const queryInterface = this.sequelize.getQueryInterface()
|
||||
const tables = await queryInterface.showAllTables()
|
||||
return Array.isArray(tables) && tables.length > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to db, build models and run migrations
|
||||
* @param {boolean} [force=false] Used for testing, drops & re-creates all tables
|
||||
*/
|
||||
async init(force = false) {
|
||||
this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite')
|
||||
|
||||
// First check if this is a new database
|
||||
this.isNew = !(await this.checkHasDb()) || force
|
||||
this.dialect = this.getConfiguredDialect()
|
||||
if (this.isSqliteDialect()) {
|
||||
this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite')
|
||||
// First check if this is a new database
|
||||
this.isNew = !(await this.checkHasDb()) || force
|
||||
} else {
|
||||
this.dbPath = process.env.DATABASE_URL || null
|
||||
this.isNew = !!force
|
||||
}
|
||||
|
||||
if (!(await this.connect())) {
|
||||
throw new Error('Database connection failed')
|
||||
}
|
||||
|
||||
if (this.isPostgresDialect() && !force) {
|
||||
this.isNew = !(await this.checkHasTables())
|
||||
}
|
||||
|
||||
try {
|
||||
const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath)
|
||||
await migrationManager.init(packageJson.version)
|
||||
|
|
@ -214,7 +260,11 @@ class Database {
|
|||
* @returns {boolean}
|
||||
*/
|
||||
async connect() {
|
||||
Logger.info(`[Database] Initializing db at "${this.dbPath}"`)
|
||||
if (this.isSqliteDialect()) {
|
||||
Logger.info(`[Database] Initializing sqlite db at "${this.dbPath}"`)
|
||||
} else {
|
||||
Logger.info(`[Database] Initializing postgres db connection`)
|
||||
}
|
||||
|
||||
let logging = false
|
||||
let benchmark = false
|
||||
|
|
@ -229,13 +279,26 @@ class Database {
|
|||
benchmark = true
|
||||
}
|
||||
|
||||
this.sequelize = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: this.dbPath,
|
||||
logging: logging,
|
||||
benchmark: benchmark,
|
||||
transactionType: 'IMMEDIATE'
|
||||
})
|
||||
if (this.isSqliteDialect()) {
|
||||
this.sequelize = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: this.dbPath,
|
||||
logging: logging,
|
||||
benchmark: benchmark,
|
||||
transactionType: 'IMMEDIATE'
|
||||
})
|
||||
} else {
|
||||
if (!process.env.DATABASE_URL) {
|
||||
Logger.error(`[Database] DATABASE_URL is required when DB_DIALECT=postgres`)
|
||||
return false
|
||||
}
|
||||
|
||||
this.sequelize = new Sequelize(process.env.DATABASE_URL, {
|
||||
dialect: 'postgres',
|
||||
logging: logging,
|
||||
benchmark: benchmark
|
||||
})
|
||||
}
|
||||
|
||||
// Helper function
|
||||
this.sequelize.uppercaseFirst = (str) => (str ? `${str[0].toUpperCase()}${str.substr(1)}` : '')
|
||||
|
|
@ -243,34 +306,36 @@ class Database {
|
|||
try {
|
||||
await this.sequelize.authenticate()
|
||||
|
||||
// Set SQLite pragmas from environment variables
|
||||
const allowedPragmas = [
|
||||
{ name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' },
|
||||
{ name: 'cache_size', env: 'SQLITE_CACHE_SIZE' },
|
||||
{ name: 'temp_store', env: 'SQLITE_TEMP_STORE' }
|
||||
]
|
||||
if (this.isSqliteDialect()) {
|
||||
// Set SQLite pragmas from environment variables
|
||||
const allowedPragmas = [
|
||||
{ name: 'mmap_size', env: 'SQLITE_MMAP_SIZE' },
|
||||
{ name: 'cache_size', env: 'SQLITE_CACHE_SIZE' },
|
||||
{ name: 'temp_store', env: 'SQLITE_TEMP_STORE' }
|
||||
]
|
||||
|
||||
for (const pragma of allowedPragmas) {
|
||||
const value = process.env[pragma.env]
|
||||
if (value !== undefined) {
|
||||
try {
|
||||
Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`)
|
||||
await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`)
|
||||
const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`)
|
||||
Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result)
|
||||
} catch (error) {
|
||||
Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error)
|
||||
for (const pragma of allowedPragmas) {
|
||||
const value = process.env[pragma.env]
|
||||
if (value !== undefined) {
|
||||
try {
|
||||
Logger.info(`[Database] Running "PRAGMA ${pragma.name} = ${value}"`)
|
||||
await this.sequelize.query(`PRAGMA ${pragma.name} = ${value}`)
|
||||
const [result] = await this.sequelize.query(`PRAGMA ${pragma.name}`)
|
||||
Logger.debug(`[Database] "PRAGMA ${pragma.name}" query result:`, result)
|
||||
} catch (error) {
|
||||
Logger.error(`[Database] Failed to set SQLite pragma ${pragma.name}`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env.NUSQLITE3_PATH) {
|
||||
await this.loadExtension(process.env.NUSQLITE3_PATH)
|
||||
Logger.info(`[Database] Db supports unaccent and unicode foldings`)
|
||||
this.supportsUnaccent = true
|
||||
this.supportsUnicodeFoldings = true
|
||||
if (process.env.NUSQLITE3_PATH) {
|
||||
await this.loadExtension(process.env.NUSQLITE3_PATH)
|
||||
Logger.info(`[Database] Db supports unaccent and unicode foldings`)
|
||||
this.supportsUnaccent = true
|
||||
this.supportsUnicodeFoldings = true
|
||||
}
|
||||
}
|
||||
Logger.info(`[Database] Db connection was successful`)
|
||||
Logger.info(`[Database] Db connection was successful (${this.dialect})`)
|
||||
return true
|
||||
} catch (error) {
|
||||
Logger.error(`[Database] Failed to connect to db`, error)
|
||||
|
|
@ -307,7 +372,7 @@ class Database {
|
|||
* Disconnect from db
|
||||
*/
|
||||
async disconnect() {
|
||||
Logger.info(`[Database] Disconnecting sqlite db`)
|
||||
Logger.info(`[Database] Disconnecting ${this.dialect} db`)
|
||||
await this.sequelize.close()
|
||||
}
|
||||
|
||||
|
|
@ -315,7 +380,7 @@ class Database {
|
|||
* Reconnect to db and init
|
||||
*/
|
||||
async reconnect() {
|
||||
Logger.info(`[Database] Reconnecting sqlite db`)
|
||||
Logger.info(`[Database] Reconnecting ${this.dialect} db`)
|
||||
await this.init()
|
||||
}
|
||||
|
||||
|
|
@ -368,7 +433,7 @@ class Database {
|
|||
* Loads most of the data from the database. This is a temporary solution.
|
||||
*/
|
||||
async loadData() {
|
||||
if (this.isNew && (await dbMigration.checkShouldMigrate())) {
|
||||
if (this.isSqliteDialect() && this.isNew && (await dbMigration.checkShouldMigrate())) {
|
||||
Logger.info(`[Database] New database was created and old database was detected - migrating old to new`)
|
||||
await dbMigration.migrate(this.models)
|
||||
}
|
||||
|
|
@ -847,6 +912,11 @@ WHERE EXISTS (
|
|||
* It adds triggers to update libraryItems.title[IgnorePrefix] when (books|podcasts).title[IgnorePrefix] is updated
|
||||
*/
|
||||
async addTriggers() {
|
||||
if (!this.isSqliteDialect()) {
|
||||
Logger.info(`[Database] Skipping sqlite-only triggers for dialect ${this.dialect}`)
|
||||
return
|
||||
}
|
||||
|
||||
await this.addTriggerIfNotExists('books', 'title', 'id', 'libraryItems', 'title', 'mediaId')
|
||||
await this.addTriggerIfNotExists('books', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId')
|
||||
await this.addTriggerIfNotExists('podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId')
|
||||
|
|
@ -954,6 +1024,7 @@ WHERE EXISTS (
|
|||
this.supportsUnaccent = supportsUnaccent
|
||||
this.query = query
|
||||
this.hasAccents = false
|
||||
this.dialect = sequelize.getDialect()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -989,9 +1060,10 @@ WHERE EXISTS (
|
|||
*/
|
||||
matchExpression(column) {
|
||||
const pattern = this.sequelize.escape(`%${this.query}%`)
|
||||
if (!this.supportsUnaccent) return `${column} LIKE ${pattern}`
|
||||
const likeOperator = this.dialect === 'postgres' ? 'ILIKE' : 'LIKE'
|
||||
if (!this.supportsUnaccent) return `${column} ${likeOperator} ${pattern}`
|
||||
const normalizedColumn = this.hasAccents ? column : this.normalize(column)
|
||||
return `${normalizedColumn} LIKE ${pattern}`
|
||||
return `${normalizedColumn} ${likeOperator} ${pattern}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -477,14 +477,27 @@ class Server {
|
|||
|
||||
// Remove series from hide from continue listening that no longer exist
|
||||
try {
|
||||
const users = await Database.sequelize.query(`SELECT u.id, u.username, u.extraData, json_group_array(value) AS seriesIdsToRemove FROM users u, json_each(u.extraData->"seriesHideFromContinueListening") LEFT JOIN series se ON se.id = value WHERE se.id IS NULL GROUP BY u.id;`, {
|
||||
model: Database.userModel,
|
||||
type: Sequelize.QueryTypes.SELECT
|
||||
const users = await Database.userModel.findAll({
|
||||
attributes: ['id', 'username', 'extraData']
|
||||
})
|
||||
|
||||
for (const user of users) {
|
||||
const extraData = JSON.parse(user.extraData)
|
||||
const existingSeriesIds = extraData.seriesHideFromContinueListening
|
||||
const seriesIdsToRemove = JSON.parse(user.dataValues.seriesIdsToRemove)
|
||||
const extraData = typeof user.extraData === 'string' ? JSON.parse(user.extraData || '{}') : user.extraData || {}
|
||||
const existingSeriesIds = Array.isArray(extraData.seriesHideFromContinueListening) ? extraData.seriesHideFromContinueListening : []
|
||||
if (!existingSeriesIds.length) continue
|
||||
|
||||
const existingSeries = await Database.seriesModel.findAll({
|
||||
attributes: ['id'],
|
||||
where: {
|
||||
id: {
|
||||
[Sequelize.Op.in]: existingSeriesIds
|
||||
}
|
||||
}
|
||||
})
|
||||
const existingSeriesSet = new Set(existingSeries.map((series) => series.id))
|
||||
const seriesIdsToRemove = existingSeriesIds.filter((seriesId) => !existingSeriesSet.has(seriesId))
|
||||
if (!seriesIdsToRemove.length) continue
|
||||
|
||||
Logger.info(`[Server] Found ${seriesIdsToRemove.length} non-existent series in seriesHideFromContinueListening for user "${user.username}" - Removing (${seriesIdsToRemove.join(',')})`)
|
||||
const newExtraData = {
|
||||
...extraData,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ const libraryFilters = require('../utils/queries/libraryFilters')
|
|||
const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters')
|
||||
const authorFilters = require('../utils/queries/authorFilters')
|
||||
const zipHelpers = require('../utils/zipHelpers')
|
||||
const { isPostgres, noCaseSortExpression, jsonArrayExpand } = require('../utils/sqlDialectHelpers')
|
||||
|
||||
/**
|
||||
* @typedef RequestUserObject
|
||||
|
|
@ -1039,9 +1040,9 @@ class LibraryController {
|
|||
let order = undefined
|
||||
const direction = payload.sortDesc ? 'DESC' : 'ASC'
|
||||
if (payload.sortBy === 'name') {
|
||||
order = [[Sequelize.literal('name COLLATE NOCASE'), direction]]
|
||||
order = [[Sequelize.literal(noCaseSortExpression('name', Database.sequelize)), direction]]
|
||||
} else if (payload.sortBy === 'lastFirst') {
|
||||
order = [[Sequelize.literal('lastFirst COLLATE NOCASE'), direction]]
|
||||
order = [[Sequelize.literal(noCaseSortExpression('lastFirst', Database.sequelize)), direction]]
|
||||
} else if (payload.sortBy === 'addedAt') {
|
||||
order = [['createdAt', direction]]
|
||||
} else if (payload.sortBy === 'updatedAt') {
|
||||
|
|
@ -1338,16 +1339,23 @@ class LibraryController {
|
|||
|
||||
const fileExt = req.query.ext === 'abs' ? 'abs' : 'json'
|
||||
const metadataFilename = `metadata.${fileExt}`
|
||||
const metadataFilenameCountQuery = isPostgres(Database.sequelize)
|
||||
? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{metadata,filename}' = :metadataFilename)`
|
||||
: `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = :metadataFilename)`
|
||||
|
||||
const libraryItemsWithMetadata = await Database.libraryItemModel.findAll({
|
||||
attributes: ['id', 'libraryFiles'],
|
||||
where: [
|
||||
{
|
||||
libraryId: req.library.id
|
||||
},
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(libraryFiles) AND json_extract(json_each.value, "$.metadata.filename") = "${metadataFilename}")`), {
|
||||
Sequelize.where(Sequelize.literal(metadataFilenameCountQuery), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
})
|
||||
]
|
||||
],
|
||||
replacements: {
|
||||
metadataFilename
|
||||
}
|
||||
})
|
||||
if (!libraryItemsWithMetadata.length) {
|
||||
Logger.info(`[LibraryController] No ${metadataFilename} files found to remove`)
|
||||
|
|
|
|||
|
|
@ -94,20 +94,26 @@ class MigrationManager {
|
|||
|
||||
// Only proceed with migration if there are migrations to run
|
||||
if (migrationsToRun.length > 0) {
|
||||
const dialect = typeof this.sequelize.getDialect === 'function' ? this.sequelize.getDialect() : 'sqlite'
|
||||
const isSqlite = !dialect || dialect === 'sqlite'
|
||||
const originalDbPath = path.join(this.configPath, 'absdatabase.sqlite')
|
||||
const backupDbPath = path.join(this.configPath, 'absdatabase.backup.sqlite')
|
||||
try {
|
||||
Logger.info(`[MigrationManager] Migrating database ${migrationDirection} to version ${this.serverVersion}`)
|
||||
Logger.info(`[MigrationManager] Migrations to run: ${migrationsToRun.join(', ')}`)
|
||||
// Create a backup copy of the SQLite database before starting migrations
|
||||
await fs.copy(originalDbPath, backupDbPath)
|
||||
Logger.info('Created a backup of the original database.')
|
||||
if (isSqlite) {
|
||||
// Create a backup copy of the SQLite database before starting migrations
|
||||
await fs.copy(originalDbPath, backupDbPath)
|
||||
Logger.info('Created a backup of the original database.')
|
||||
}
|
||||
|
||||
// Run migrations
|
||||
await this.umzug[migrationDirection]({ migrations: migrationsToRun, rerun: 'ALLOW' })
|
||||
|
||||
// Clean up the backup
|
||||
await fs.remove(backupDbPath)
|
||||
if (isSqlite) {
|
||||
// Clean up the backup
|
||||
await fs.remove(backupDbPath)
|
||||
}
|
||||
|
||||
Logger.info('[MigrationManager] Migrations successfully applied to the original database.')
|
||||
} catch (error) {
|
||||
|
|
@ -115,13 +121,15 @@ class MigrationManager {
|
|||
|
||||
await this.sequelize.close()
|
||||
|
||||
// Step 3: If migration fails, save the failed original and restore the backup
|
||||
const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite')
|
||||
await fs.move(originalDbPath, failedDbPath, { overwrite: true })
|
||||
Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.')
|
||||
if (isSqlite) {
|
||||
// Step 3: If migration fails, save the failed original and restore the backup
|
||||
const failedDbPath = path.join(this.configPath, 'absdatabase.failed.sqlite')
|
||||
await fs.move(originalDbPath, failedDbPath, { overwrite: true })
|
||||
Logger.info('[MigrationManager] Saved the failed database as absdatabase.failed.sqlite.')
|
||||
|
||||
await fs.move(backupDbPath, originalDbPath, { overwrite: true })
|
||||
Logger.info('[MigrationManager] Restored the original database from the backup.')
|
||||
await fs.move(backupDbPath, originalDbPath, { overwrite: true })
|
||||
Logger.info('[MigrationManager] Restored the original database from the backup.')
|
||||
}
|
||||
|
||||
Logger.info('[MigrationManager] Migration failed. Exiting Audiobookshelf with code 1.')
|
||||
process.exit(1)
|
||||
|
|
@ -191,16 +199,15 @@ class MigrationManager {
|
|||
}
|
||||
|
||||
async fetchVersionsFromDatabase() {
|
||||
const migrationsMetaTable = `"${MigrationManager.MIGRATIONS_META_TABLE}"`
|
||||
await this.checkOrCreateMigrationsMetaTable()
|
||||
|
||||
const [{ version }] = await this.sequelize.query("SELECT value as version FROM :migrationsMeta WHERE key = 'version'", {
|
||||
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
const [{ version }] = await this.sequelize.query(`SELECT value as version FROM ${migrationsMetaTable} WHERE key = 'version'`, {
|
||||
type: Sequelize.QueryTypes.SELECT
|
||||
})
|
||||
this.databaseVersion = version
|
||||
|
||||
const [{ maxVersion }] = await this.sequelize.query("SELECT value as maxVersion FROM :migrationsMeta WHERE key = 'maxVersion'", {
|
||||
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
const [{ maxVersion }] = await this.sequelize.query(`SELECT value as maxVersion FROM ${migrationsMetaTable} WHERE key = 'maxVersion'`, {
|
||||
type: Sequelize.QueryTypes.SELECT
|
||||
})
|
||||
this.maxVersion = maxVersion
|
||||
|
|
@ -212,8 +219,7 @@ class MigrationManager {
|
|||
|
||||
// If the table exists, check that the `version` and `maxVersion` rows exist
|
||||
if (migrationsMetaTableExists) {
|
||||
const [{ count }] = await this.sequelize.query("SELECT COUNT(*) as count FROM :migrationsMeta WHERE key IN ('version', 'maxVersion')", {
|
||||
replacements: { migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
const [{ count }] = await this.sequelize.query(`SELECT COUNT(*) as count FROM "${MigrationManager.MIGRATIONS_META_TABLE}" WHERE key IN ('version', 'maxVersion')`, {
|
||||
type: Sequelize.QueryTypes.SELECT
|
||||
})
|
||||
if (count < 2) {
|
||||
|
|
@ -241,8 +247,8 @@ class MigrationManager {
|
|||
allowNull: false
|
||||
}
|
||||
})
|
||||
await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", {
|
||||
replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
await this.sequelize.query(`INSERT INTO "${MigrationManager.MIGRATIONS_META_TABLE}" (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')`, {
|
||||
replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0' },
|
||||
type: Sequelize.QueryTypes.INSERT
|
||||
})
|
||||
Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`)
|
||||
|
|
@ -299,8 +305,8 @@ class MigrationManager {
|
|||
|
||||
async updateMaxVersion() {
|
||||
try {
|
||||
await this.sequelize.query("UPDATE :migrationsMeta SET value = :maxVersion WHERE key = 'maxVersion'", {
|
||||
replacements: { maxVersion: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :maxVersion WHERE key = 'maxVersion'`, {
|
||||
replacements: { maxVersion: this.serverVersion },
|
||||
type: Sequelize.QueryTypes.UPDATE
|
||||
})
|
||||
} catch (error) {
|
||||
|
|
@ -311,8 +317,8 @@ class MigrationManager {
|
|||
|
||||
async updateDatabaseVersion() {
|
||||
try {
|
||||
await this.sequelize.query("UPDATE :migrationsMeta SET value = :version WHERE key = 'version'", {
|
||||
replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
|
||||
await this.sequelize.query(`UPDATE "${MigrationManager.MIGRATIONS_META_TABLE}" SET value = :version WHERE key = 'version'`, {
|
||||
replacements: { version: this.serverVersion },
|
||||
type: Sequelize.QueryTypes.UPDATE
|
||||
})
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -39,6 +39,11 @@ const authorsJoin = `${authors} JOIN ${bookAuthors} ON ${authors}.id = ${bookAut
|
|||
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
|
||||
*/
|
||||
async function up({ context: { queryInterface, logger } }) {
|
||||
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
|
||||
logger.info(`${loggerPrefix} skipping sqlite-specific migration on non-sqlite dialect`)
|
||||
return
|
||||
}
|
||||
|
||||
const helper = new MigrationHelper(queryInterface, logger)
|
||||
|
||||
// Upwards migration script
|
||||
|
|
@ -72,6 +77,11 @@ async function up({ context: { queryInterface, logger } }) {
|
|||
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
|
||||
*/
|
||||
async function down({ context: { queryInterface, logger } }) {
|
||||
if (queryInterface.sequelize.getDialect() !== 'sqlite') {
|
||||
logger.info(`${loggerPrefix} skipping sqlite-specific rollback on non-sqlite dialect`)
|
||||
return
|
||||
}
|
||||
|
||||
// Downward migration script
|
||||
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
|
||||
|
||||
|
|
|
|||
|
|
@ -128,6 +128,8 @@ class Author extends Model {
|
|||
* @param {import('../Database').sequelize} sequelize
|
||||
*/
|
||||
static init(sequelize) {
|
||||
const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' }
|
||||
|
||||
super.init(
|
||||
{
|
||||
id: {
|
||||
|
|
@ -146,12 +148,7 @@ class Author extends Model {
|
|||
modelName: 'author',
|
||||
indexes: [
|
||||
{
|
||||
fields: [
|
||||
{
|
||||
name: 'name',
|
||||
collate: 'NOCASE'
|
||||
}
|
||||
]
|
||||
fields: [nameIndexField]
|
||||
},
|
||||
// {
|
||||
// fields: [{
|
||||
|
|
|
|||
|
|
@ -137,6 +137,8 @@ class Book extends Model {
|
|||
* @param {import('../Database').sequelize} sequelize
|
||||
*/
|
||||
static init(sequelize) {
|
||||
const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' }
|
||||
|
||||
super.init(
|
||||
{
|
||||
id: {
|
||||
|
|
@ -171,12 +173,7 @@ class Book extends Model {
|
|||
modelName: 'book',
|
||||
indexes: [
|
||||
{
|
||||
fields: [
|
||||
{
|
||||
name: 'title',
|
||||
collate: 'NOCASE'
|
||||
}
|
||||
]
|
||||
fields: [titleIndexField]
|
||||
},
|
||||
// {
|
||||
// fields: [{
|
||||
|
|
|
|||
|
|
@ -662,6 +662,11 @@ class LibraryItem extends Model {
|
|||
* @param {import('../Database').sequelize} sequelize
|
||||
*/
|
||||
static init(sequelize) {
|
||||
const titleIndexField = sequelize.getDialect() === 'postgres' ? 'title' : { name: 'title', collate: 'NOCASE' }
|
||||
const titleIgnorePrefixIndexField = sequelize.getDialect() === 'postgres' ? 'titleIgnorePrefix' : { name: 'titleIgnorePrefix', collate: 'NOCASE' }
|
||||
const authorNamesFirstLastIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesFirstLast' : { name: 'authorNamesFirstLast', collate: 'NOCASE' }
|
||||
const authorNamesLastFirstIndexField = sequelize.getDialect() === 'postgres' ? 'authorNamesLastFirst' : { name: 'authorNamesLastFirst', collate: 'NOCASE' }
|
||||
|
||||
super.init(
|
||||
{
|
||||
id: {
|
||||
|
|
@ -710,16 +715,16 @@ class LibraryItem extends Model {
|
|||
fields: ['libraryId', 'mediaType', 'createdAt']
|
||||
},
|
||||
{
|
||||
fields: ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }]
|
||||
fields: ['libraryId', 'mediaType', titleIndexField]
|
||||
},
|
||||
{
|
||||
fields: ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }]
|
||||
fields: ['libraryId', 'mediaType', titleIgnorePrefixIndexField]
|
||||
},
|
||||
{
|
||||
fields: ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }]
|
||||
fields: ['libraryId', 'mediaType', authorNamesFirstLastIndexField]
|
||||
},
|
||||
{
|
||||
fields: ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }]
|
||||
fields: ['libraryId', 'mediaType', authorNamesLastFirstIndexField]
|
||||
},
|
||||
{
|
||||
fields: ['libraryId', 'mediaId', 'mediaType']
|
||||
|
|
|
|||
|
|
@ -87,6 +87,8 @@ class Series extends Model {
|
|||
* @param {import('../Database').sequelize} sequelize
|
||||
*/
|
||||
static init(sequelize) {
|
||||
const nameIndexField = sequelize.getDialect() === 'postgres' ? 'name' : { name: 'name', collate: 'NOCASE' }
|
||||
|
||||
super.init(
|
||||
{
|
||||
id: {
|
||||
|
|
@ -103,12 +105,7 @@ class Series extends Model {
|
|||
modelName: 'series',
|
||||
indexes: [
|
||||
{
|
||||
fields: [
|
||||
{
|
||||
name: 'name',
|
||||
collate: 'NOCASE'
|
||||
}
|
||||
]
|
||||
fields: [nameIndexField]
|
||||
},
|
||||
// {
|
||||
// fields: [{
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ const LibraryItemScanner = require('./LibraryItemScanner')
|
|||
const LibraryScan = require('./LibraryScan')
|
||||
const LibraryItemScanData = require('./LibraryItemScanData')
|
||||
const Task = require('../objects/Task')
|
||||
const { isPostgres, jsonArrayExpand } = require('../utils/sqlDialectHelpers')
|
||||
|
||||
class LibraryScanner {
|
||||
constructor() {
|
||||
|
|
@ -674,12 +675,16 @@ async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingle
|
|||
// check if it was moved from another folder by comparing the ino to the library files
|
||||
const ino = await fileUtils.getIno(fullPath)
|
||||
if (!ino) return null
|
||||
const inodeMatchQuery = isPostgres(Database.sequelize)
|
||||
? `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_each.value #>> '{ino}' = :inode)`
|
||||
: `(SELECT count(*) FROM ${jsonArrayExpand('libraryFiles', Database.sequelize, { textValues: false })} WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)`
|
||||
|
||||
const existingLibraryItem = await Database.libraryItemModel.findOneExpanded(
|
||||
[
|
||||
{
|
||||
libraryId: libraryId
|
||||
},
|
||||
sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), {
|
||||
sequelize.where(sequelize.literal(inodeMatchQuery), {
|
||||
[sequelize.Op.gt]: 0
|
||||
})
|
||||
],
|
||||
|
|
|
|||
243
server/scripts/migrateSqliteToPostgres.js
Normal file
243
server/scripts/migrateSqliteToPostgres.js
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
const sqlite3 = require('sqlite3')
|
||||
const { Client } = require('pg')
|
||||
|
||||
const SQLITE_PATH = process.env.SQLITE_PATH || '/config/absdatabase.sqlite'
|
||||
const DATABASE_URL = process.env.DATABASE_URL
|
||||
const PG_SCHEMA = process.env.PG_SCHEMA || 'public'
|
||||
const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500)
|
||||
const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true'
|
||||
|
||||
if (!DATABASE_URL) {
|
||||
console.error('[migrate] DATABASE_URL is required')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const preferredOrder = [
|
||||
'migrationsMeta',
|
||||
'SequelizeMeta',
|
||||
'settings',
|
||||
'users',
|
||||
'apiKeys',
|
||||
'sessions',
|
||||
'libraries',
|
||||
'libraryFolders',
|
||||
'authors',
|
||||
'series',
|
||||
'books',
|
||||
'podcasts',
|
||||
'podcastEpisodes',
|
||||
'libraryItems',
|
||||
'bookAuthors',
|
||||
'bookSeries',
|
||||
'collections',
|
||||
'collectionBooks',
|
||||
'playlists',
|
||||
'playlistMediaItems',
|
||||
'mediaProgresses',
|
||||
'devices',
|
||||
'playbackSessions',
|
||||
'feeds',
|
||||
'feedEpisodes',
|
||||
'mediaItemShares',
|
||||
'customMetadataProviders'
|
||||
]
|
||||
|
||||
function quoteIdent(identifier) {
|
||||
return `"${String(identifier).replace(/"/g, '""')}"`
|
||||
}
|
||||
|
||||
function openSqlite(filePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const db = new sqlite3.Database(filePath, sqlite3.OPEN_READONLY, (err) => {
|
||||
if (err) return reject(err)
|
||||
resolve(db)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function sqliteAll(db, sql, params = []) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.all(sql, params, (err, rows) => {
|
||||
if (err) return reject(err)
|
||||
resolve(rows)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function sqliteGet(db, sql, params = []) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.get(sql, params, (err, row) => {
|
||||
if (err) return reject(err)
|
||||
resolve(row)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function normalizeBoolean(value) {
|
||||
if (value === null || value === undefined) return null
|
||||
if (typeof value === 'boolean') return value
|
||||
if (typeof value === 'number') return value !== 0
|
||||
if (typeof value === 'string') {
|
||||
const v = value.trim().toLowerCase()
|
||||
return v === 'true' || v === '1' || v === 't'
|
||||
}
|
||||
return !!value
|
||||
}
|
||||
|
||||
function normalizeJson(value) {
|
||||
if (value === null || value === undefined || value === '') return null
|
||||
if (typeof value === 'object') return value
|
||||
try {
|
||||
return JSON.parse(value)
|
||||
} catch (error) {
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
function convertValue(value, pgColumn) {
|
||||
if (!pgColumn) return value
|
||||
const dataType = pgColumn.data_type
|
||||
const udtName = pgColumn.udt_name
|
||||
|
||||
if (dataType === 'boolean') {
|
||||
return normalizeBoolean(value)
|
||||
}
|
||||
|
||||
if (dataType === 'json' || dataType === 'jsonb' || udtName === 'json' || udtName === 'jsonb') {
|
||||
return normalizeJson(value)
|
||||
}
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log(`[migrate] sqlite source: ${SQLITE_PATH}`)
|
||||
console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`)
|
||||
console.log(`[migrate] dry run: ${DRY_RUN}`)
|
||||
|
||||
const sqliteDb = await openSqlite(SQLITE_PATH)
|
||||
const pg = new Client({ connectionString: DATABASE_URL })
|
||||
await pg.connect()
|
||||
|
||||
try {
|
||||
const sqliteTablesRows = await sqliteAll(
|
||||
sqliteDb,
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
||||
)
|
||||
const sqliteTables = sqliteTablesRows.map((row) => row.name)
|
||||
|
||||
const pgTablesRows = await pg.query(
|
||||
`SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`,
|
||||
[PG_SCHEMA]
|
||||
)
|
||||
const pgTables = new Set(pgTablesRows.rows.map((row) => row.table_name))
|
||||
|
||||
const tablesToMigrate = sqliteTables.filter((table) => pgTables.has(table))
|
||||
tablesToMigrate.sort((a, b) => {
|
||||
const ai = preferredOrder.indexOf(a)
|
||||
const bi = preferredOrder.indexOf(b)
|
||||
if (ai === -1 && bi === -1) return a.localeCompare(b)
|
||||
if (ai === -1) return 1
|
||||
if (bi === -1) return -1
|
||||
return ai - bi
|
||||
})
|
||||
|
||||
if (!tablesToMigrate.length) {
|
||||
throw new Error('No overlapping tables found between SQLite and PostgreSQL')
|
||||
}
|
||||
|
||||
console.log(`[migrate] tables to migrate: ${tablesToMigrate.join(', ')}`)
|
||||
|
||||
const pgColumnsByTable = new Map()
|
||||
for (const table of tablesToMigrate) {
|
||||
const columnsResult = await pg.query(
|
||||
`SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`,
|
||||
[PG_SCHEMA, table]
|
||||
)
|
||||
const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name, column]))
|
||||
pgColumnsByTable.set(table, columnMap)
|
||||
}
|
||||
|
||||
if (!DRY_RUN) {
|
||||
await pg.query('BEGIN')
|
||||
await pg.query('SET session_replication_role = replica')
|
||||
|
||||
const truncateList = tablesToMigrate.map((table) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`).join(', ')
|
||||
await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`)
|
||||
console.log('[migrate] truncated target tables')
|
||||
}
|
||||
|
||||
for (const table of tablesToMigrate) {
|
||||
const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(table)}`)
|
||||
const pgColumns = pgColumnsByTable.get(table)
|
||||
const insertColumns = rows.length ? Object.keys(rows[0]).filter((column) => pgColumns.has(column)) : []
|
||||
|
||||
if (!rows.length || !insertColumns.length) {
|
||||
console.log(`[migrate] ${table}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`)
|
||||
continue
|
||||
}
|
||||
|
||||
if (!DRY_RUN) {
|
||||
for (let offset = 0; offset < rows.length; offset += BATCH_SIZE) {
|
||||
const batchRows = rows.slice(offset, offset + BATCH_SIZE)
|
||||
const valuesSql = []
|
||||
const params = []
|
||||
let paramIndex = 1
|
||||
|
||||
for (const row of batchRows) {
|
||||
const placeholders = []
|
||||
for (const column of insertColumns) {
|
||||
const pgColumn = pgColumns.get(column)
|
||||
params.push(convertValue(row[column], pgColumn))
|
||||
placeholders.push(`$${paramIndex++}`)
|
||||
}
|
||||
valuesSql.push(`(${placeholders.join(', ')})`)
|
||||
}
|
||||
|
||||
const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)} (${insertColumns.map(quoteIdent).join(', ')}) VALUES ${valuesSql.join(', ')}`
|
||||
await pg.query(insertSql, params)
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[migrate] ${table}: ${rows.length} rows`)
|
||||
}
|
||||
|
||||
if (!DRY_RUN) {
|
||||
await pg.query('SET session_replication_role = DEFAULT')
|
||||
await pg.query('COMMIT')
|
||||
console.log('[migrate] migration transaction committed')
|
||||
}
|
||||
|
||||
const parity = []
|
||||
for (const table of tablesToMigrate) {
|
||||
const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(table)}`)
|
||||
const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`)
|
||||
parity.push({
|
||||
table,
|
||||
sqliteCount: Number(sqliteCountRow.count || 0),
|
||||
postgresCount: Number(pgCountResult.rows[0].count || 0)
|
||||
})
|
||||
}
|
||||
|
||||
const mismatches = parity.filter((row) => row.sqliteCount !== row.postgresCount)
|
||||
if (mismatches.length) {
|
||||
console.error('[migrate] row-count mismatches detected:')
|
||||
mismatches.forEach((row) => {
|
||||
console.error(`[migrate] ${row.table}: sqlite=${row.sqliteCount} postgres=${row.postgresCount}`)
|
||||
})
|
||||
process.exitCode = 2
|
||||
} else {
|
||||
console.log('[migrate] parity check passed for all migrated tables')
|
||||
}
|
||||
} finally {
|
||||
sqliteDb.close()
|
||||
await pg.end()
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('[migrate] failed:', error)
|
||||
process.exit(1)
|
||||
})
|
||||
|
|
@ -5,6 +5,7 @@ const libraryItemsBookFilters = require('./libraryItemsBookFilters')
|
|||
const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters')
|
||||
const { createNewSortInstance } = require('../../libs/fastSort')
|
||||
const { profile } = require('../../utils/profiler')
|
||||
const { booleanLiteral, jsonArrayContainsAny } = require('../sqlDialectHelpers')
|
||||
const naturalSort = createNewSortInstance({
|
||||
comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
|
||||
})
|
||||
|
|
@ -235,13 +236,13 @@ module.exports = {
|
|||
if (userPermissionBookWhere.bookWhere.length) {
|
||||
let attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id'
|
||||
if (!user.canAccessExplicitContent) {
|
||||
attrQuery += ' AND b.explicit = 0'
|
||||
attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}`
|
||||
}
|
||||
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
|
||||
if (user.permissions.selectedTagsNotAccessible) {
|
||||
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0'
|
||||
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0`
|
||||
} else {
|
||||
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0'
|
||||
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0`
|
||||
}
|
||||
}
|
||||
seriesWhere.push(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ const Sequelize = require('sequelize')
|
|||
const Database = require('../../Database')
|
||||
const libraryItemsBookFilters = require('./libraryItemsBookFilters')
|
||||
const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters')
|
||||
const { jsonArrayContainsAny } = require('../sqlDialectHelpers')
|
||||
|
||||
module.exports = {
|
||||
/**
|
||||
|
|
@ -12,7 +13,7 @@ module.exports = {
|
|||
async getAllLibraryItemsWithTags(tags) {
|
||||
const libraryItems = []
|
||||
const booksWithTag = await Database.bookModel.findAll({
|
||||
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), {
|
||||
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
replacements: {
|
||||
|
|
@ -46,7 +47,7 @@ module.exports = {
|
|||
libraryItems.push(libraryItem)
|
||||
}
|
||||
const podcastsWithTag = await Database.podcastModel.findAll({
|
||||
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:tags))`), {
|
||||
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'tags', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
replacements: {
|
||||
|
|
@ -77,7 +78,7 @@ module.exports = {
|
|||
async getAllLibraryItemsWithGenres(genres) {
|
||||
const libraryItems = []
|
||||
const booksWithGenre = await Database.bookModel.findAll({
|
||||
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), {
|
||||
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
replacements: {
|
||||
|
|
@ -107,7 +108,7 @@ module.exports = {
|
|||
libraryItems.push(libraryItem)
|
||||
}
|
||||
const podcastsWithGenre = await Database.podcastModel.findAll({
|
||||
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(genres) WHERE json_valid(genres) AND json_each.value IN (:genres))`), {
|
||||
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('genres', 'genres', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
replacements: {
|
||||
|
|
@ -138,7 +139,7 @@ module.exports = {
|
|||
async getAllLibraryItemsWithNarrators(narrators) {
|
||||
const libraryItems = []
|
||||
const booksWithGenre = await Database.bookModel.findAll({
|
||||
where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(narrators) WHERE json_valid(narrators) AND json_each.value IN (:narrators))`), {
|
||||
where: Sequelize.where(Sequelize.literal(jsonArrayContainsAny('narrators', 'narrators', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
replacements: {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ const authorFilters = require('./authorFilters')
|
|||
const ShareManager = require('../../managers/ShareManager')
|
||||
const { profile } = require('../profiler')
|
||||
const stringifySequelizeQuery = require('../stringifySequelizeQuery')
|
||||
const { booleanLiteral, noCaseSortExpression, coalesceFunctionName, jsonArrayContainsAny, jsonArrayContainsValue, jsonArrayExpand } = require('../sqlDialectHelpers')
|
||||
const countCache = new Map()
|
||||
|
||||
module.exports = {
|
||||
|
|
@ -27,10 +28,10 @@ module.exports = {
|
|||
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
|
||||
replacements['userTagsSelected'] = user.permissions.itemTagsSelected
|
||||
if (user.permissions.selectedTagsNotAccessible) {
|
||||
bookWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0))
|
||||
bookWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0))
|
||||
} else {
|
||||
bookWhere.push(
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), {
|
||||
Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
})
|
||||
)
|
||||
|
|
@ -189,7 +190,7 @@ module.exports = {
|
|||
} else if (group === 'explicit') {
|
||||
mediaWhere['explicit'] = true
|
||||
} else if (['genres', 'tags', 'narrators'].includes(group)) {
|
||||
mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), {
|
||||
mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
})
|
||||
replacements.filterValue = value
|
||||
|
|
@ -256,9 +257,9 @@ module.exports = {
|
|||
|
||||
const getTitleOrder = () => {
|
||||
if (global.ServerSettings.sortingIgnorePrefix) {
|
||||
return [Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]
|
||||
return [Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]
|
||||
} else {
|
||||
return [Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]
|
||||
return [Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -273,20 +274,23 @@ module.exports = {
|
|||
} else if (sortBy === 'media.duration') {
|
||||
return [['duration', dir]]
|
||||
} else if (sortBy === 'media.metadata.publishedYear') {
|
||||
return [[Sequelize.literal(`CAST(\`book\`.\`publishedYear\` AS INTEGER)`), dir]]
|
||||
return [[Sequelize.literal('CAST(book.publishedYear AS INTEGER)'), dir]]
|
||||
} else if (sortBy === 'media.metadata.authorNameLF') {
|
||||
// Sort by author name last first, secondary sort by title
|
||||
return [[Sequelize.literal('`libraryItem`.`authorNamesLastFirst` COLLATE NOCASE'), dir], getTitleOrder()]
|
||||
return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesLastFirst', Database.sequelize)), dir], getTitleOrder()]
|
||||
} else if (sortBy === 'media.metadata.authorName') {
|
||||
// Sort by author name first last, secondary sort by title
|
||||
return [[Sequelize.literal('`libraryItem`.`authorNamesFirstLast` COLLATE NOCASE'), dir], getTitleOrder()]
|
||||
return [[Sequelize.literal(noCaseSortExpression('libraryItem.authorNamesFirstLast', Database.sequelize)), dir], getTitleOrder()]
|
||||
} else if (sortBy === 'media.metadata.title') {
|
||||
if (collapseseries) {
|
||||
return [[Sequelize.literal('display_title COLLATE NOCASE'), dir]]
|
||||
return [[Sequelize.literal(noCaseSortExpression('display_title', Database.sequelize)), dir]]
|
||||
}
|
||||
return [getTitleOrder()]
|
||||
} else if (sortBy === 'sequence') {
|
||||
const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'
|
||||
if (Database.sequelize.getDialect() === 'postgres') {
|
||||
return [[Sequelize.literal(`CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ${nullDir}`)]]
|
||||
}
|
||||
return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]]
|
||||
} else if (sortBy === 'progress') {
|
||||
return [[Sequelize.literal(`mediaProgresses.updatedAt ${dir} NULLS LAST`)]]
|
||||
|
|
@ -326,7 +330,13 @@ module.exports = {
|
|||
required: true
|
||||
}
|
||||
],
|
||||
order: [Sequelize.literal('CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')]
|
||||
order: [
|
||||
Sequelize.literal(
|
||||
Database.sequelize.getDialect() === 'postgres'
|
||||
? 'CAST("books.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST'
|
||||
: 'CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST'
|
||||
)
|
||||
]
|
||||
})
|
||||
const bookSeriesToInclude = []
|
||||
const booksToInclude = []
|
||||
|
|
@ -455,12 +465,28 @@ module.exports = {
|
|||
})
|
||||
} else if (filterGroup === 'ebooks' && filterValue === 'supplementary') {
|
||||
// TODO: Temp workaround for filtering supplementary ebook
|
||||
libraryItemWhere['libraryFiles'] = {
|
||||
[Sequelize.Op.substring]: `"isSupplementary":true`
|
||||
if (Database.sequelize.getDialect() === 'postgres') {
|
||||
libraryItemWhere[Sequelize.Op.and] = [
|
||||
Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), {
|
||||
[Sequelize.Op.like]: '%"isSupplementary":true%'
|
||||
})
|
||||
]
|
||||
} else {
|
||||
libraryItemWhere['libraryFiles'] = {
|
||||
[Sequelize.Op.substring]: `"isSupplementary":true`
|
||||
}
|
||||
}
|
||||
} else if (filterGroup === 'ebooks' && filterValue === 'no-supplementary') {
|
||||
libraryItemWhere['libraryFiles'] = {
|
||||
[Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`)
|
||||
if (Database.sequelize.getDialect() === 'postgres') {
|
||||
libraryItemWhere[Sequelize.Op.and] = [
|
||||
Sequelize.where(Sequelize.literal('CAST("libraryItem"."libraryFiles" AS TEXT)'), {
|
||||
[Sequelize.Op.notLike]: '%"isSupplementary":true%'
|
||||
})
|
||||
]
|
||||
} else {
|
||||
libraryItemWhere['libraryFiles'] = {
|
||||
[Sequelize.Op.notLike]: Sequelize.literal(`\'%"isSupplementary":true%\'`)
|
||||
}
|
||||
}
|
||||
} else if (filterGroup === 'missing' && filterValue === 'authors') {
|
||||
authorInclude = {
|
||||
|
|
@ -502,7 +528,13 @@ module.exports = {
|
|||
})
|
||||
if (sortBy !== 'sequence') {
|
||||
// Secondary sort by sequence
|
||||
sortOrder.push([Sequelize.literal('CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')])
|
||||
sortOrder.push([
|
||||
Sequelize.literal(
|
||||
Database.sequelize.getDialect() === 'postgres'
|
||||
? 'CAST("series.bookSeries"."sequence" AS DOUBLE PRECISION) ASC NULLS LAST'
|
||||
: 'CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST'
|
||||
)
|
||||
])
|
||||
}
|
||||
} else if (filterGroup === 'issues') {
|
||||
libraryItemWhere[Sequelize.Op.or] = [
|
||||
|
|
@ -594,10 +626,14 @@ module.exports = {
|
|||
|
||||
// When collapsing series and sorting by title then use the series name instead of the book title
|
||||
// for this set an attribute "display_title" to use in sorting
|
||||
const fallbackLibraryItemTitle = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."title"' : '`libraryItem`.`title`'
|
||||
const fallbackLibraryItemTitleIgnorePrefix = Database.sequelize.getDialect() === 'postgres' ? '"libraryItem"."titleIgnorePrefix"' : '`libraryItem`.`titleIgnorePrefix`'
|
||||
const fallbackFn = coalesceFunctionName(Database.sequelize)
|
||||
|
||||
if (global.ServerSettings.sortingIgnorePrefix) {
|
||||
bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`titleIgnorePrefix\`)`), 'display_title'])
|
||||
bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.nameIgnorePrefix FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitleIgnorePrefix})`), 'display_title'])
|
||||
} else {
|
||||
bookAttributes.include.push([Sequelize.literal(`IFNULL((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), \`libraryItem\`.\`title\`)`), 'display_title'])
|
||||
bookAttributes.include.push([Sequelize.literal(`${fallbackFn}((SELECT s.name FROM bookSeries AS bs, series AS s WHERE bs.seriesId = s.id AND bs.bookId = book.id AND bs.id IN (${bookSeriesToInclude.map((v) => `"${v.id}"`).join(', ')})), ${fallbackLibraryItemTitle})`), 'display_title'])
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -716,13 +752,14 @@ module.exports = {
|
|||
bookWhere.push(...userPermissionBookWhere.bookWhere)
|
||||
|
||||
let includeAttributes = [[Sequelize.literal('(SELECT max(mp.updatedAt) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id)'), 'recent_progress']]
|
||||
let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = 0 OR mp.isFinished IS NULL)`
|
||||
let booksNotFinishedQuery = `SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId AND mp.userId = :userId WHERE bs.seriesId = series.id AND (mp.isFinished = ${booleanLiteral(false, Database.sequelize)} OR mp.isFinished IS NULL)`
|
||||
|
||||
if (library.settings.onlyShowLaterBooksInContinueSeries) {
|
||||
const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as FLOAT) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = 1 AND mp.userId = :userId AND bs.seriesId = series.id)`
|
||||
const castType = Database.sequelize.getDialect() === 'postgres' ? 'DOUBLE PRECISION' : 'FLOAT'
|
||||
const maxSequenceQuery = `(SELECT CAST(max(bs.sequence) as ${castType}) FROM bookSeries bs, mediaProgresses mp WHERE mp.mediaItemId = bs.bookId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)} AND mp.userId = :userId AND bs.seriesId = series.id)`
|
||||
includeAttributes.push([Sequelize.literal(`${maxSequenceQuery}`), 'maxSequence'])
|
||||
|
||||
booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as FLOAT) > ${maxSequenceQuery}`
|
||||
booksNotFinishedQuery = booksNotFinishedQuery + ` AND CAST(bs.sequence as ${castType}) > ${maxSequenceQuery}`
|
||||
}
|
||||
|
||||
const { rows: series, count } = await Database.seriesModel.findAndCountAll({
|
||||
|
|
@ -735,7 +772,7 @@ module.exports = {
|
|||
},
|
||||
// TODO: Simplify queries
|
||||
// Has at least 1 book finished
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = 1)`), {
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE bs.seriesId = series.id AND mp.mediaItemId = bs.bookId AND mp.userId = :userId AND mp.isFinished = ${booleanLiteral(true, Database.sequelize)})`), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
// Has at least 1 book not finished (that has a sequence number higher than the highest already read, if library config is toggled)
|
||||
|
|
@ -743,7 +780,7 @@ module.exports = {
|
|||
[Sequelize.Op.gte]: 1
|
||||
}),
|
||||
// Has no books in progress
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = 0 AND mp.currentTime > 0)`), 0)
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM mediaProgresses mp, bookSeries bs WHERE mp.mediaItemId = bs.bookId AND mp.userId = :userId AND bs.seriesId = series.id AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)} AND mp.currentTime > 0)`), 0)
|
||||
],
|
||||
attributes: {
|
||||
include: includeAttributes
|
||||
|
|
@ -760,7 +797,7 @@ module.exports = {
|
|||
order: [[Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]],
|
||||
where: {
|
||||
'$book.mediaProgresses.isFinished$': {
|
||||
[Sequelize.Op.or]: [null, 0]
|
||||
[Sequelize.Op.or]: [null, false]
|
||||
}
|
||||
},
|
||||
include: {
|
||||
|
|
@ -854,7 +891,7 @@ module.exports = {
|
|||
{
|
||||
libraryId
|
||||
},
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = 1 OR mp.currentTime > 0))`), 0)
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = bs.bookId WHERE bs.seriesId = series.id AND mp.userId = :userId AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0))`), 0)
|
||||
],
|
||||
replacements: {
|
||||
userId: user.id,
|
||||
|
|
@ -904,6 +941,7 @@ module.exports = {
|
|||
id: {
|
||||
[Sequelize.Op.in]: booksFromSeriesToInclude
|
||||
}
|
||||
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -1130,7 +1168,7 @@ module.exports = {
|
|||
|
||||
// Search narrators
|
||||
const narratorMatches = []
|
||||
const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
|
||||
const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, ${jsonArrayExpand('b.narrators', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
|
||||
replacements: {
|
||||
libraryId: library.id,
|
||||
limit,
|
||||
|
|
@ -1147,7 +1185,7 @@ module.exports = {
|
|||
|
||||
// Search tags
|
||||
const tagMatches = []
|
||||
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.tags', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
replacements: {
|
||||
libraryId: library.id,
|
||||
limit,
|
||||
|
|
@ -1164,7 +1202,7 @@ module.exports = {
|
|||
|
||||
// Search genres
|
||||
const genreMatches = []
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
replacements: {
|
||||
libraryId: library.id,
|
||||
limit,
|
||||
|
|
@ -1244,7 +1282,7 @@ module.exports = {
|
|||
*/
|
||||
async getGenresWithCount(libraryId) {
|
||||
const genres = []
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, ${jsonArrayExpand('b.genres', Database.sequelize)} WHERE b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
|
||||
replacements: {
|
||||
libraryId
|
||||
},
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ const Database = require('../../Database')
|
|||
const Logger = require('../../Logger')
|
||||
const { profile } = require('../../utils/profiler')
|
||||
const stringifySequelizeQuery = require('../stringifySequelizeQuery')
|
||||
const { jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression, jsonArrayExpand, jsonPathNumber } = require('../sqlDialectHelpers')
|
||||
|
||||
const countCache = new Map()
|
||||
|
||||
|
|
@ -24,10 +25,10 @@ module.exports = {
|
|||
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
|
||||
replacements['userTagsSelected'] = user.permissions.itemTagsSelected
|
||||
if (user.permissions.selectedTagsNotAccessible) {
|
||||
podcastWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0))
|
||||
podcastWhere.push(Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), 0))
|
||||
} else {
|
||||
podcastWhere.push(
|
||||
Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), {
|
||||
Sequelize.where(Sequelize.literal(jsonArrayContainsAny('tags', 'userTagsSelected', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
})
|
||||
)
|
||||
|
|
@ -53,7 +54,7 @@ module.exports = {
|
|||
const replacements = {}
|
||||
|
||||
if (['genres', 'tags'].includes(group)) {
|
||||
mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), {
|
||||
mediaWhere[group] = Sequelize.where(Sequelize.literal(jsonArrayContainsValue(group, 'filterValue', Database.sequelize)), {
|
||||
[Sequelize.Op.gte]: 1
|
||||
})
|
||||
replacements.filterValue = value
|
||||
|
|
@ -87,12 +88,12 @@ module.exports = {
|
|||
return [[Sequelize.literal('libraryItem.mtime'), dir]]
|
||||
} else if (sortBy === 'media.metadata.author') {
|
||||
const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'
|
||||
return [[Sequelize.literal(`\`podcast\`.\`author\` COLLATE NOCASE ${nullDir}`)]]
|
||||
return [[Sequelize.literal(`${noCaseSortExpression('podcast.author', Database.sequelize)} ${nullDir}`)]]
|
||||
} else if (sortBy === 'media.metadata.title') {
|
||||
if (global.ServerSettings.sortingIgnorePrefix) {
|
||||
return [[Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]]
|
||||
return [[Sequelize.literal(noCaseSortExpression('libraryItem.titleIgnorePrefix', Database.sequelize)), dir]]
|
||||
} else {
|
||||
return [[Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]]
|
||||
return [[Sequelize.literal(noCaseSortExpression('libraryItem.title', Database.sequelize)), dir]]
|
||||
}
|
||||
} else if (sortBy === 'media.numTracks') {
|
||||
return [['numEpisodes', dir]]
|
||||
|
|
@ -455,7 +456,7 @@ module.exports = {
|
|||
|
||||
// Search tags
|
||||
const tagMatches = []
|
||||
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.tags', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
replacements: {
|
||||
libraryId: library.id,
|
||||
limit,
|
||||
|
|
@ -472,7 +473,7 @@ module.exports = {
|
|||
|
||||
// Search genres
|
||||
const genreMatches = []
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
|
||||
replacements: {
|
||||
libraryId: library.id,
|
||||
limit,
|
||||
|
|
@ -563,12 +564,12 @@ module.exports = {
|
|||
* @returns {Promise<{ totalSize:number, totalDuration:number, numAudioFiles:number, totalItems:number}>}
|
||||
*/
|
||||
async getPodcastLibraryStats(libraryId) {
|
||||
const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = "podcast" AND li.libraryId = :libraryId;`, {
|
||||
const [sizeResults] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize FROM libraryItems li WHERE li.mediaType = 'podcast' AND li.libraryId = :libraryId;`, {
|
||||
replacements: {
|
||||
libraryId
|
||||
}
|
||||
})
|
||||
const [statResults] = await Database.sequelize.query(`SELECT SUM(json_extract(pe.audioFile, '$.duration')) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, {
|
||||
const [statResults] = await Database.sequelize.query(`SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) AS totalDuration, COUNT(DISTINCT(li.id)) AS totalItems, COUNT(pe.id) AS numAudioFiles FROM libraryItems li, podcasts p LEFT OUTER JOIN podcastEpisodes pe ON pe.podcastId = p.id WHERE p.id = li.mediaId AND li.libraryId = :libraryId;`, {
|
||||
replacements: {
|
||||
libraryId
|
||||
}
|
||||
|
|
@ -588,7 +589,7 @@ module.exports = {
|
|||
*/
|
||||
async getGenresWithCount(libraryId) {
|
||||
const genres = []
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
|
||||
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, ${jsonArrayExpand('p.genres', Database.sequelize)} WHERE p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC;`, {
|
||||
replacements: {
|
||||
libraryId
|
||||
},
|
||||
|
|
@ -611,7 +612,7 @@ module.exports = {
|
|||
*/
|
||||
async getLongestPodcasts(libraryId, limit) {
|
||||
const podcasts = await Database.podcastModel.findAll({
|
||||
attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(json_extract(pe.audioFile, '$.duration')) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']],
|
||||
attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(${jsonPathNumber('pe.audioFile', ['duration'], Database.sequelize)}) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']],
|
||||
include: {
|
||||
model: Database.libraryItemModel,
|
||||
attributes: ['id', 'libraryId'],
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ const Sequelize = require('sequelize')
|
|||
const Logger = require('../../Logger')
|
||||
const Database = require('../../Database')
|
||||
const libraryItemsBookFilters = require('./libraryItemsBookFilters')
|
||||
const { booleanLiteral, jsonArrayContainsAny, jsonArrayContainsValue, noCaseSortExpression } = require('../sqlDialectHelpers')
|
||||
|
||||
module.exports = {
|
||||
decode(text) {
|
||||
|
|
@ -60,7 +61,7 @@ module.exports = {
|
|||
// TODO: Simplify and break-out
|
||||
let attrQuery = null
|
||||
if (['genres', 'tags', 'narrators'].includes(filterGroup)) {
|
||||
attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (SELECT count(*) FROM json_each(b.${filterGroup}) WHERE json_valid(b.${filterGroup}) AND json_each.value = :filterValue) > 0`
|
||||
attrQuery = `SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ${jsonArrayContainsValue(`b.${filterGroup}`, 'filterValue', Database.sequelize)} > 0`
|
||||
userPermissionBookWhere.replacements.filterValue = filterValue
|
||||
} else if (filterGroup === 'authors') {
|
||||
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs, bookAuthors ba WHERE bs.seriesId = series.id AND bs.bookId = b.id AND ba.bookId = b.id AND ba.authorId = :filterValue'
|
||||
|
|
@ -73,18 +74,18 @@ module.exports = {
|
|||
userPermissionBookWhere.replacements.filterValue = filterValue
|
||||
} else if (filterGroup === 'progress') {
|
||||
if (filterValue === 'not-finished') {
|
||||
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)'
|
||||
attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})`
|
||||
userPermissionBookWhere.replacements.userId = user.id
|
||||
} else if (filterValue === 'finished') {
|
||||
const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = 0)'
|
||||
const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished IS NULL OR mp.isFinished = ${booleanLiteral(false, Database.sequelize)})`
|
||||
seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0))
|
||||
userPermissionBookWhere.replacements.userId = user.id
|
||||
} else if (filterValue === 'not-started') {
|
||||
const progQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = 1 OR mp.currentTime > 0)'
|
||||
const progQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.isFinished = ${booleanLiteral(true, Database.sequelize)} OR mp.currentTime > 0)`
|
||||
seriesWhere.push(Sequelize.where(Sequelize.literal(`(${progQuery})`), 0))
|
||||
userPermissionBookWhere.replacements.userId = user.id
|
||||
} else if (filterValue === 'in-progress') {
|
||||
attrQuery = 'SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = 0'
|
||||
attrQuery = `SELECT count(*) FROM books b, bookSeries bs LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = b.id AND mp.userId = :userId WHERE bs.seriesId = series.id AND bs.bookId = b.id AND (mp.currentTime > 0 OR mp.ebookProgress > 0) AND mp.isFinished = ${booleanLiteral(false, Database.sequelize)}`
|
||||
userPermissionBookWhere.replacements.userId = user.id
|
||||
}
|
||||
}
|
||||
|
|
@ -95,13 +96,13 @@ module.exports = {
|
|||
if (!attrQuery) attrQuery = 'SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id'
|
||||
|
||||
if (!user.canAccessExplicitContent) {
|
||||
attrQuery += ' AND b.explicit = 0'
|
||||
attrQuery += ` AND b.explicit = ${booleanLiteral(false, Database.sequelize)}`
|
||||
}
|
||||
if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
|
||||
if (user.permissions.selectedTagsNotAccessible) {
|
||||
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0'
|
||||
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} = 0`
|
||||
} else {
|
||||
attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0'
|
||||
attrQuery += ` AND ${jsonArrayContainsAny('b.tags', 'userTagsSelected', Database.sequelize)} > 0`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -128,9 +129,9 @@ module.exports = {
|
|||
order.push(['createdAt', dir])
|
||||
} else if (sortBy === 'name') {
|
||||
if (global.ServerSettings.sortingIgnorePrefix) {
|
||||
order.push([Sequelize.literal('nameIgnorePrefix COLLATE NOCASE'), dir])
|
||||
order.push([Sequelize.literal(noCaseSortExpression('nameIgnorePrefix', Database.sequelize)), dir])
|
||||
} else {
|
||||
order.push([Sequelize.literal('`series`.`name` COLLATE NOCASE'), dir])
|
||||
order.push([Sequelize.literal(noCaseSortExpression('series.name', Database.sequelize)), dir])
|
||||
}
|
||||
} else if (sortBy === 'totalDuration') {
|
||||
seriesAttributes.include.push([Sequelize.literal('(SELECT SUM(b.duration) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND b.id = bs.bookId)'), 'totalDuration'])
|
||||
|
|
|
|||
75
server/utils/sqlDialectHelpers.js
Normal file
75
server/utils/sqlDialectHelpers.js
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
function getDialect(sequelize) {
|
||||
if (!sequelize) return 'sqlite'
|
||||
if (typeof sequelize.getDialect === 'function') return sequelize.getDialect()
|
||||
return sequelize.dialect?.name || 'sqlite'
|
||||
}
|
||||
|
||||
function isPostgres(sequelize) {
|
||||
return getDialect(sequelize) === 'postgres'
|
||||
}
|
||||
|
||||
function booleanLiteral(value, sequelize) {
|
||||
if (isPostgres(sequelize)) return value ? 'TRUE' : 'FALSE'
|
||||
return value ? '1' : '0'
|
||||
}
|
||||
|
||||
function noCaseSortExpression(columnExpression, sequelize) {
|
||||
if (isPostgres(sequelize)) return `LOWER(${columnExpression})`
|
||||
return `${columnExpression} COLLATE NOCASE`
|
||||
}
|
||||
|
||||
function coalesceFunctionName(sequelize) {
|
||||
return isPostgres(sequelize) ? 'COALESCE' : 'IFNULL'
|
||||
}
|
||||
|
||||
function jsonArrayContainsAny(columnExpression, bindName, sequelize) {
|
||||
if (isPostgres(sequelize)) {
|
||||
return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:${bindName}))`
|
||||
}
|
||||
return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value IN (:${bindName}))`
|
||||
}
|
||||
|
||||
function jsonArrayContainsValue(columnExpression, bindName, sequelize) {
|
||||
if (isPostgres(sequelize)) {
|
||||
return `(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :${bindName})`
|
||||
}
|
||||
return `(SELECT count(*) FROM json_each(${columnExpression}) WHERE json_valid(${columnExpression}) AND json_each.value = :${bindName})`
|
||||
}
|
||||
|
||||
function jsonArrayExpand(columnExpression, sequelize, options = {}) {
|
||||
const alias = options.alias || 'json_each'
|
||||
const textValues = options.textValues !== false
|
||||
if (isPostgres(sequelize)) {
|
||||
const fn = textValues ? 'jsonb_array_elements_text' : 'jsonb_array_elements'
|
||||
return `${fn}(COALESCE(${columnExpression}::jsonb, '[]'::jsonb)) AS ${alias}(value)`
|
||||
}
|
||||
return `json_each(${columnExpression})`
|
||||
}
|
||||
|
||||
function jsonPathText(columnExpression, pathSegments, sequelize) {
|
||||
const path = Array.isArray(pathSegments) ? pathSegments : [pathSegments]
|
||||
if (isPostgres(sequelize)) {
|
||||
return `${columnExpression}::jsonb #>> '{${path.join(',')}}'`
|
||||
}
|
||||
return `json_extract(${columnExpression}, '$.${path.join('.')}')`
|
||||
}
|
||||
|
||||
function jsonPathNumber(columnExpression, pathSegments, sequelize) {
|
||||
if (isPostgres(sequelize)) {
|
||||
return `NULLIF(${jsonPathText(columnExpression, pathSegments, sequelize)}, '')::double precision`
|
||||
}
|
||||
return `json_extract(${columnExpression}, '$.${[].concat(pathSegments).join('.')}')`
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getDialect,
|
||||
isPostgres,
|
||||
booleanLiteral,
|
||||
noCaseSortExpression,
|
||||
coalesceFunctionName,
|
||||
jsonArrayContainsAny,
|
||||
jsonArrayContainsValue,
|
||||
jsonArrayExpand,
|
||||
jsonPathText,
|
||||
jsonPathNumber
|
||||
}
|
||||
58
test/server/utils/sqlDialectHelpers.test.js
Normal file
58
test/server/utils/sqlDialectHelpers.test.js
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
const { expect } = require('chai')
|
||||
|
||||
const {
|
||||
booleanLiteral,
|
||||
noCaseSortExpression,
|
||||
coalesceFunctionName,
|
||||
jsonArrayContainsAny,
|
||||
jsonArrayContainsValue,
|
||||
jsonArrayExpand,
|
||||
jsonPathText,
|
||||
jsonPathNumber
|
||||
} = require('../../../server/utils/sqlDialectHelpers')
|
||||
|
||||
const sqlite = {
|
||||
getDialect: () => 'sqlite'
|
||||
}
|
||||
|
||||
const postgres = {
|
||||
getDialect: () => 'postgres'
|
||||
}
|
||||
|
||||
describe('sqlDialectHelpers', () => {
|
||||
it('should return sqlite and postgres boolean literals', () => {
|
||||
expect(booleanLiteral(true, sqlite)).to.equal('1')
|
||||
expect(booleanLiteral(false, sqlite)).to.equal('0')
|
||||
expect(booleanLiteral(true, postgres)).to.equal('TRUE')
|
||||
expect(booleanLiteral(false, postgres)).to.equal('FALSE')
|
||||
})
|
||||
|
||||
it('should generate case-insensitive sort expressions', () => {
|
||||
expect(noCaseSortExpression('name', sqlite)).to.equal('name COLLATE NOCASE')
|
||||
expect(noCaseSortExpression('name', postgres)).to.equal('LOWER(name)')
|
||||
})
|
||||
|
||||
it('should choose the correct null-coalescing function name', () => {
|
||||
expect(coalesceFunctionName(sqlite)).to.equal('IFNULL')
|
||||
expect(coalesceFunctionName(postgres)).to.equal('COALESCE')
|
||||
})
|
||||
|
||||
it('should generate array membership count queries per dialect', () => {
|
||||
expect(jsonArrayContainsAny('tags', 'selectedTags', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:selectedTags))')
|
||||
expect(jsonArrayContainsAny('tags', 'selectedTags', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value IN (:selectedTags))")
|
||||
|
||||
expect(jsonArrayContainsValue('tags', 'tag', sqlite)).to.equal('(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value = :tag)')
|
||||
expect(jsonArrayContainsValue('tags', 'tag', postgres)).to.equal("(SELECT count(*) FROM jsonb_array_elements_text(COALESCE(tags::jsonb, '[]'::jsonb)) AS json_each(value) WHERE json_each.value = :tag)")
|
||||
})
|
||||
|
||||
it('should generate json array expansion and path extraction by dialect', () => {
|
||||
expect(jsonArrayExpand('books.tags', sqlite)).to.equal('json_each(books.tags)')
|
||||
expect(jsonArrayExpand('books.tags', postgres)).to.equal("jsonb_array_elements_text(COALESCE(books.tags::jsonb, '[]'::jsonb)) AS json_each(value)")
|
||||
|
||||
expect(jsonPathText('payload', ['metadata', 'filename'], sqlite)).to.equal("json_extract(payload, '$.metadata.filename')")
|
||||
expect(jsonPathText('payload', ['metadata', 'filename'], postgres)).to.equal("payload::jsonb #>> '{metadata,filename}'")
|
||||
|
||||
expect(jsonPathNumber('payload', ['duration'], sqlite)).to.equal("json_extract(payload, '$.duration')")
|
||||
expect(jsonPathNumber('payload', ['duration'], postgres)).to.equal("NULLIF(payload::jsonb #>> '{duration}', '')::double precision")
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue