groove/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2026-08-03 15:11:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add postgres-native backup support

Browse source 
Before, scheduled backups opened the Postgres URL through sqlite3, causing a fatal SQLITE_CANTOPEN restart and leaving no usable database backup.

After, Postgres uses pg_dump and pg_restore archives while SQLite keeps its existing format, and SQLite open failures are contained.
This commit is contained in:
Kevin Gatera 2026-08-02 17:55:46 -04:00
parent 4195e8800e
commit 07d5450dc7
No known key found for this signature in database
GPG key ID: F0D9F5932458CFB9
4 changed files with 367 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Dockerfile
View file

@ -50,6 +50,7 @@ ARG NUSQLITE3_PATH
RUN apk add --no-cache --update \
tzdata \
ffmpeg \
postgresql-client \
tini
WORKDIR /app

274
server/managers/BackupManager.js
View file

@ -1,3 +1,4 @@
const childProcess = require('child_process')
const sqlite3 = require('sqlite3')
const Path = require('path')
const Logger = require('../Logger')
@ -47,6 +48,30 @@ class BackupManager {
return global.ServerSettings.maxBackupSize || Infinity
}
get databaseBackupConfig() {
if (Database.isPostgresDialect()) {
return {
dialect: 'postgres',
entryName: 'absdatabase.postgres.dump'
}
}
return {
dialect: 'sqlite',
entryName: 'absdatabase.sqlite'
}
}
getBackupDialect(backup) {
if (backup.key === 'postgres') return 'postgres'
if (backup.key === 'sqlite' || !backup.key) return 'sqlite'
return null
}
getBackupEntryName(dialect) {
return dialect === 'postgres' ? 'absdatabase.postgres.dump' : 'absdatabase.sqlite'
}
async init() {
try {
const backupsDirExists = await fs.pathExists(this.backupPath)
@ -130,13 +155,6 @@ class BackupManager {
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(400).send('Failed to read backup file - backup might not be a valid .zip file')
}
if (!entries['absdatabase.sqlite']) {
Logger.error(`[BackupManager] Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.`)
await zip.close().catch(() => {})
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(500).send('Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.')
}
const detailsEntry = entries['details']
if (!detailsEntry) {
Logger.error('[BackupManager] Invalid backup - missing details entry')
@ -151,10 +169,26 @@ class BackupManager {
return res.status(400).send('Invalid backup file - details entry too large')
}
const data = await zip.entryData('details')
const details = data.toString('utf8').split('\n')
let backup
try {
const data = await zip.entryData('details')
const details = data.toString('utf8').split('\n')
backup = new Backup({ details, fullPath: tempPath })
} catch (error) {
Logger.error(`[BackupManager] Invalid backup with no readable details file`, tempPath, error)
await zip.close().catch(() => {})
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(400).send('Invalid backup file. Missing readable details.')
}
const backup = new Backup({ details, fullPath: tempPath })
const backupDialect = this.getBackupDialect(backup)
const databaseEntryName = this.getBackupEntryName(backupDialect)
if (!backupDialect || !entries[databaseEntryName]) {
Logger.error(`[BackupManager] Invalid backup with no ${databaseEntryName} file - unsupported database backup.`)
await zip.close().catch(() => {})
await fs.remove(tempPath).catch((err) => Logger.error(`[BackupManager] Failed to remove rejected backup file "${tempPath}"`, err))
return res.status(500).send(`Invalid backup file. Does not include ${databaseEntryName}.`)
}
if (!backup.serverVersion) {
Logger.error(`[BackupManager] Invalid backup with no server version - might be a backup created before version 2.0.0`)
@ -204,8 +238,24 @@ class BackupManager {
const entries = await zip.entries()
const backupDialect = this.getBackupDialect(backup)
const currentDialect = Database.isPostgresDialect() ? 'postgres' : 'sqlite'
if (!backupDialect) {
await zip.close()
return res.status(500).send('Invalid backup file. Unsupported database backup format.')
}
if (backupDialect !== currentDialect) {
await zip.close()
return res.status(400).send(`Cannot apply a ${backupDialect} backup while using the ${currentDialect} database.`)
}
if (backupDialect === 'postgres') {
return this.requestApplyPostgresBackup(apiCacheManager, backup, zip, entries, res)
}
// Ensure backup has an absdatabase.sqlite file
if (!Object.keys(entries).includes('absdatabase.sqlite')) {
if (!Object.keys(entries).includes(this.getBackupEntryName(backupDialect))) {
Logger.error(`[BackupManager] Cannot apply old backup ${backup.fullPath}`)
await zip.close()
return res.status(500).send('Invalid backup file. Does not include absdatabase.sqlite. This might be from an older Audiobookshelf server.')
@ -266,6 +316,70 @@ class BackupManager {
SocketAuthority.emitter('backup_applied')
}
async requestApplyPostgresBackup(apiCacheManager, backup, zip, entries, res) {
const databaseEntryName = this.getBackupEntryName('postgres')
if (!Object.keys(entries).includes(databaseEntryName)) {
Logger.error(`[BackupManager] Cannot apply Postgres backup ${backup.fullPath}`)
await zip.close()
return res.status(500).send(`Invalid backup file. Does not include ${databaseEntryName}.`)
}
const tempDumpPath = Path.join(global.ConfigPath, 'absdatabase-postgres-temp.dump')
let reconnected = false
let zipClosed = false
const closeZip = async () => {
if (zipClosed) return
zipClosed = true
await zip.close()
}
try {
await fs.remove(tempDumpPath)
await zip.extract(databaseEntryName, tempDumpPath)
if (!(await fs.pathExists(tempDumpPath))) {
await closeZip()
return res.status(500).send('Failed to extract Postgres database dump from backup')
}
await Database.disconnect()
await this.restorePostgresDb(tempDumpPath)
await fs.ensureDir(this.ItemsMetadataPath)
await zip.extract('metadata-items/', this.ItemsMetadataPath)
await fs.ensureDir(this.AuthorsMetadataPath)
await zip.extract('metadata-authors/', this.AuthorsMetadataPath)
await closeZip()
await Database.reconnect()
reconnected = true
await apiCacheManager.reset()
await CacheManager.purgeAll()
res.sendStatus(200)
SocketAuthority.emitter('backup_applied')
} catch (error) {
Logger.error(`[BackupManager] Failed to apply Postgres backup`, error)
try {
await closeZip()
} catch (closeError) {
Logger.error(`[BackupManager] Failed to close Postgres backup archive`, closeError)
}
if (!reconnected) {
try {
await Database.reconnect()
} catch (reconnectError) {
Logger.error(`[BackupManager] Failed to reconnect after Postgres backup apply`, reconnectError)
}
}
return res.status(500).send(`Failed to apply Postgres backup: ${error?.message || 'Unknown Error'}`)
} finally {
await fs.remove(tempDumpPath)
}
}
async loadBackups() {
try {
const filesInDir = await fs.readdir(this.backupPath)
@ -303,6 +417,14 @@ class BackupManager {
const details = data.toString('utf8').split('\n')
const backup = new Backup({ details, fullPath: fullFilePath })
const backupDialect = this.getBackupDialect(backup)
const databaseEntryName = this.getBackupEntryName(backupDialect)
if (!backupDialect || !Object.keys(await zip.entries()).includes(databaseEntryName)) {
Logger.error(`[BackupManager] Unsupported database backup format found "${backup.filename}"`)
await zip.close()
continue
}
if (!backup.serverVersion) {
// Backups before v2
@ -333,33 +455,34 @@ class BackupManager {
async runBackup() {
// Check if Metadata Path is inside Config Path (otherwise there will be an infinite loop as the archiver tries to zip itself)
Logger.info(`[BackupManager] Running Backup`)
const databaseBackupConfig = this.databaseBackupConfig
const newBackup = new Backup()
newBackup.setData(this.backupPath)
newBackup.setData(this.backupPath, databaseBackupConfig.dialect)
await fs.ensureDir(this.AuthorsMetadataPath)
// Create backup sqlite file
const sqliteBackupPath = await this.backupSqliteDb(newBackup).catch((error) => {
Logger.error(`[BackupManager] Failed to backup sqlite db`, error)
// Create a database dump
const databaseBackupPath = await this.backupDatabase(newBackup).catch((error) => {
Logger.error(`[BackupManager] Failed to backup ${databaseBackupConfig.dialect} database`, error)
const errorMsg = error?.message || error || 'Unknown Error'
NotificationManager.onBackupFailed(errorMsg)
return false
})
if (!sqliteBackupPath) {
if (!databaseBackupPath) {
return false
}
// Zip sqlite file, /metadata/items, and /metadata/authors folders
const zipResult = await this.zipBackup(sqliteBackupPath, newBackup).catch((error) => {
// Zip database dump, /metadata/items, and /metadata/authors folders
const zipResult = await this.zipBackup(databaseBackupPath, newBackup, databaseBackupConfig.entryName).catch((error) => {
Logger.error(`[BackupManager] Backup Failed ${error}`)
const errorMsg = error?.message || error || 'Unknown Error'
NotificationManager.onBackupFailed(errorMsg)
return false
})
// Remove sqlite backup
await fs.remove(sqliteBackupPath)
// Remove temporary database dump
await fs.remove(databaseBackupPath)
if (!zipResult) return false
@ -390,6 +513,10 @@ class BackupManager {
return true
}
backupDatabase(backup) {
return this.databaseBackupConfig.dialect === 'postgres' ? this.backupPostgresDb(backup) : this.backupSqliteDb(backup)
}
async removeBackup(backup) {
try {
Logger.debug(`[BackupManager] Removing Backup "${backup.fullPath}"`)
@ -406,29 +533,106 @@ class BackupManager {
* @param {Backup} backup
*/
backupSqliteDb(backup) {
const db = new sqlite3.Database(Database.dbPath)
const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.sqlite`)
return new Promise(async (resolve, reject) => {
const backup = db.backup(dbFilePath)
backup.step(-1)
backup.finish()
let db
let sqliteBackup
let settled = false
// Max time ~2 mins
for (let i = 0; i < 240; i++) {
if (backup.completed) {
return resolve(dbFilePath)
} else if (backup.failed) {
return reject(backup.message || 'Unknown failure reason')
const finish = (error, result) => {
if (settled) return
settled = true
if (db) {
db.close(() => {
if (error) reject(error)
else resolve(result)
})
} else if (error) {
reject(error)
} else {
resolve(result)
}
await new Promise((r) => setTimeout(r, 500))
}
Logger.error(`[BackupManager] Backup sqlite timed out`)
reject('Backup timed out')
const pollBackup = async () => {
// Max time ~2 mins
for (let i = 0; i < 240; i++) {
if (sqliteBackup.completed) {
return finish(null, dbFilePath)
} else if (sqliteBackup.failed) {
return finish(sqliteBackup.message || 'Unknown failure reason')
}
await new Promise((r) => setTimeout(r, 500))
}
Logger.error(`[BackupManager] Backup sqlite timed out`)
finish('Backup timed out')
}
const startBackup = () => {
try {
sqliteBackup = db.backup(dbFilePath)
sqliteBackup.step(-1)
sqliteBackup.finish()
pollBackup().catch(finish)
} catch (error) {
finish(error)
}
}
db = new sqlite3.Database(Database.dbPath, (error) => {
if (error) return finish(error)
startBackup()
})
db.on('error', finish)
})
}
zipBackup(sqliteBackupPath, backup) {
backupPostgresDb(backup) {
const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.postgres.dump`)
return this.runPostgresCommand('pg_dump', [
'--format=custom',
'--no-owner',
'--no-acl',
'--file',
dbFilePath,
'--dbname',
Database.dbPath
])
.then(() => dbFilePath)
.catch(async (error) => {
await fs.remove(dbFilePath)
throw error
})
}
restorePostgresDb(dbFilePath) {
return this.runPostgresCommand('pg_restore', [
'--clean',
'--if-exists',
'--exit-on-error',
'--single-transaction',
'--no-owner',
'--no-acl',
'--dbname',
Database.dbPath,
dbFilePath
])
}
runPostgresCommand(command, args) {
return new Promise((resolve, reject) => {
childProcess.execFile(command, args, { maxBuffer: 10 * 1024 * 1024 }, (error, stdout, stderr) => {
if (error) {
error.stderr = stderr
return reject(error)
}
resolve({ stdout, stderr })
})
})
}
zipBackup(databaseBackupPath, backup, databaseEntryName = 'absdatabase.sqlite') {
return new Promise((resolve, reject) => {
// create a file to stream archive data to
const output = fs.createWriteStream(backup.fullPath)
@ -492,7 +696,7 @@ class BackupManager {
// pipe archive data to the file
archive.pipe(output)
archive.file(sqliteBackupPath, { name: 'absdatabase.sqlite' })
archive.file(databaseBackupPath, { name: databaseEntryName })
archive.directory(this.ItemsMetadataPath, 'metadata-items')
archive.directory(this.AuthorsMetadataPath, 'metadata-authors')

6
server/objects/Backup.js
View file

@ -62,9 +62,9 @@ class Backup {
}
}
setData(backupDirPath) {
setData(backupDirPath, dialect = 'sqlite') {
this.id = date.format(new Date(), 'YYYY-MM-DD[T]HHmm')
this.key = 'sqlite'
this.key = dialect
this.datePretty = date.format(new Date(), 'ddd, MMM D YYYY HH:mm')
this.backupDirPath = backupDirPath
@ -78,4 +78,4 @@ class Backup {
this.createdAt = Date.now()
}
}
module.exports = Backup
module.exports = Backup

124
test/server/managers/BackupManager.test.js Normal file
View file

@ -0,0 +1,124 @@
const { expect } = require('chai')
const sinon = require('sinon')
const os = require('os')
const Path = require('path')
const EventEmitter = require('events')
const childProcess = require('child_process')
const sqlite3 = require('sqlite3')
const BackupManager = require('../../../server/managers/BackupManager')
const Backup = require('../../../server/objects/Backup')
const Database = require('../../../server/Database')
describe('BackupManager', () => {
let originalDialect
let originalDbPath
let originalConfigPath
let originalMetadataPath
beforeEach(() => {
originalDialect = Database.dialect
originalDbPath = Database.dbPath
originalConfigPath = global.ConfigPath
originalMetadataPath = global.MetadataPath
global.MetadataPath = os.tmpdir()
})
afterEach(() => {
Database.dialect = originalDialect
Database.dbPath = originalDbPath
global.ConfigPath = originalConfigPath
global.MetadataPath = originalMetadataPath
sinon.restore()
})
it('should select Postgres custom-format backups for the Postgres dialect', () => {
Database.dialect = 'postgres'
const manager = new BackupManager()
expect(manager.databaseBackupConfig).to.deep.equal({
dialect: 'postgres',
entryName: 'absdatabase.postgres.dump'
})
})
it('should create Postgres dumps with pg_dump and preserve the connection URL', async () => {
Database.dialect = 'postgres'
Database.dbPath = 'postgresql://localhost:5432/audiobookshelf'
global.ConfigPath = os.tmpdir()
const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => {
callback(null, '', '')
})
const manager = new BackupManager()
const backup = new Backup()
backup.id = '2026-08-02T0130'
const dumpPath = await manager.backupPostgresDb(backup)
expect(dumpPath).to.equal(Path.join(os.tmpdir(), 'absdatabase.2026-08-02T0130.postgres.dump'))
expect(execFileStub.calledOnce).to.equal(true)
expect(execFileStub.firstCall.args[0]).to.equal('pg_dump')
expect(execFileStub.firstCall.args[1]).to.deep.equal([
'--format=custom',
'--no-owner',
'--no-acl',
'--file',
dumpPath,
'--dbname',
Database.dbPath
])
})
it('should restore Postgres dumps in one transaction and clean existing objects', async () => {
Database.dialect = 'postgres'
Database.dbPath = 'postgresql://localhost:5432/audiobookshelf'
const execFileStub = sinon.stub(childProcess, 'execFile').callsFake((command, args, options, callback) => {
callback(null, '', '')
})
const manager = new BackupManager()
await manager.restorePostgresDb('/config/absdatabase-postgres-temp.dump')
expect(execFileStub.firstCall.args[0]).to.equal('pg_restore')
expect(execFileStub.firstCall.args[1]).to.deep.equal([
'--clean',
'--if-exists',
'--exit-on-error',
'--single-transaction',
'--no-owner',
'--no-acl',
'--dbname',
Database.dbPath,
'/config/absdatabase-postgres-temp.dump'
])
})
it('should reject SQLite backup open errors without an uncaught sqlite event', async () => {
Database.dialect = 'sqlite'
Database.dbPath = '/config/absdatabase.sqlite'
global.ConfigPath = os.tmpdir()
sinon.stub(sqlite3, 'Database').callsFake(function (_dbPath, callback) {
const db = new EventEmitter()
db.close = (closeCallback) => closeCallback()
process.nextTick(() => callback(Object.assign(new Error('unable to open database file'), { code: 'SQLITE_CANTOPEN' })))
return db
})
const manager = new BackupManager()
const backup = new Backup()
backup.id = '2026-08-02T0130'
let error
try {
await manager.backupSqliteDb(backup)
} catch (caughtError) {
error = caughtError
}
expect(error).to.be.an('error')
expect(error.code).to.equal('SQLITE_CANTOPEN')
})
})