Fix ebook url #75, download other files #75, fix book icon disappearing #88, backups #87

server/ApiController.js

@@ -7,7 +7,7 @@ const { isObject } = require('./utils/index')
 const Library = require('./objects/Library')
 
 class ApiController {
-  constructor(MetadataPath, db, scanner, auth, streamManager, rssFeeds, downloadManager, coverController, watcher, emitter, clientEmitter) {
+  constructor(MetadataPath, db, scanner, auth, streamManager, rssFeeds, downloadManager, coverController, backupManager, watcher, emitter, clientEmitter) {
     this.db = db
     this.scanner = scanner
     this.auth = auth
@@ -15,6 +15,7 @@ class ApiController {
     this.rssFeeds = rssFeeds
     this.downloadManager = downloadManager
     this.coverController = coverController
+    this.backupManager = backupManager
     this.watcher = watcher
     this.emitter = emitter
     this.clientEmitter = clientEmitter
@@ -61,6 +62,9 @@ class ApiController {
 
     this.router.patch('/serverSettings', this.updateServerSettings.bind(this))
 
+    this.router.delete('/backup/:id', this.deleteBackup.bind(this))
+    this.router.post('/backup/upload', this.uploadBackup.bind(this))
+
     this.router.post('/authorize', this.authorize.bind(this))
 
     this.router.get('/genres', this.getGenres.bind(this))
@@ -569,6 +573,31 @@ class ApiController {
     })
   }
 
+  async deleteBackup(req, res) {
+    if (!req.user.isRoot) {
+      Logger.error(`[ApiController] Non-Root user attempting to delete backup`, req.user)
+      return res.sendStatus(403)
+    }
+    var backup = this.backupManager.backups.find(b => b.id === req.params.id)
+    if (!backup) {
+      return res.sendStatus(404)
+    }
+    await this.backupManager.removeBackup(backup)
+    res.json(this.backupManager.backups.map(b => b.toJSON()))
+  }
+
+  async uploadBackup(req, res) {
+    if (!req.user.isRoot) {
+      Logger.error(`[ApiController] Non-Root user attempting to upload backup`, req.user)
+      return res.sendStatus(403)
+    }
+    if (!req.files.file) {
+      Logger.error('[ApiController] Upload backup invalid')
+      return res.sendStatus(500)
+    }
+    this.backupManager.uploadBackup(req, res)
+  }
+
   async download(req, res) {
     if (!req.user.canDownload) {
       Logger.error('User attempting to download without permission', req.user)

server/BackupManager.js

@@ -0,0 +1,277 @@
+const Path = require('path')
+
+const cron = require('node-cron')
+const fs = require('fs-extra')
+const archiver = require('archiver')
+const StreamZip = require('node-stream-zip')
+
+// Utils
+const { getFileSize } = require('./utils/fileUtils')
+const filePerms = require('./utils/filePerms')
+const Logger = require('./Logger')
+
+const Backup = require('./objects/Backup')
+
+class BackupManager {
+  constructor(MetadataPath, Uid, Gid, db) {
+    this.MetadataPath = MetadataPath
+    this.BackupPath = Path.join(this.MetadataPath, 'backups')
+
+    this.Uid = Uid
+    this.Gid = Gid
+    this.db = db
+
+    this.backups = []
+  }
+
+  get serverSettings() {
+    return this.db.serverSettings || {}
+  }
+
+  async init(overrideCron = null) {
+    var backupsDirExists = await fs.pathExists(this.BackupPath)
+    if (!backupsDirExists) {
+      await fs.ensureDir(this.BackupPath)
+      await filePerms(this.BackupPath, 0o774, this.Uid, this.Gid)
+    }
+
+    await this.loadBackups()
+
+    if (!this.serverSettings.backupSchedule) {
+      Logger.info(`[BackupManager] Auto Backups are disabled`)
+      return
+    }
+    try {
+      var cronSchedule = overrideCron || this.serverSettings.backupSchedule
+      cron.schedule(cronSchedule, this.runBackup.bind(this))
+    } catch (error) {
+      Logger.error(`[BackupManager] Failed to schedule backup cron ${this.serverSettings.backupSchedule}`)
+    }
+  }
+
+  async uploadBackup(req, res) {
+    var backupFile = req.files.file
+    if (Path.extname(backupFile.name) !== '.audiobookshelf') {
+      Logger.error(`[BackupManager] Invalid backup file uploaded "${backupFile.name}"`)
+      return res.status(500).send('Invalid backup file')
+    }
+
+    var tempPath = Path.join(this.BackupPath, backupFile.name)
+    var success = await backupFile.mv(tempPath).then(() => true).catch((error) => {
+      Logger.error('[BackupManager] Failed to move backup file', path, error)
+      return false
+    })
+    if (!success) {
+      return res.status(500).send('Failed to move backup file into backups directory')
+    }
+
+    const zip = new StreamZip.async({ file: tempPath })
+    const data = await zip.entryData('details')
+    var details = data.toString('utf8').split('\n')
+
+    var backup = new Backup({ details, fullPath: tempPath })
+    backup.fileSize = await getFileSize(backup.fullPath)
+
+    var existingBackupIndex = this.backups.findIndex(b => b.id === backup.id)
+    if (existingBackupIndex >= 0) {
+      Logger.warn(`[BackupManager] Backup already exists with id ${backup.id} - overwriting`)
+      this.backups.splice(existingBackupIndex, 1, backup)
+    } else {
+      this.backups.push(backup)
+    }
+
+    return res.json(this.backups.map(b => b.toJSON()))
+  }
+
+  async requestCreateBackup(socket) {
+    // Only Root User allowed
+    var client = socket.sheepClient
+    if (!client || !client.user) {
+      Logger.error(`[BackupManager] Invalid user attempting to create backup`)
+      socket.emit('backup_complete', false)
+      return
+    } else if (!client.user.isRoot) {
+      Logger.error(`[BackupManager] Non-Root user attempting to create backup`)
+      socket.emit('backup_complete', false)
+      return
+    }
+
+    var backupSuccess = await this.runBackup()
+    socket.emit('backup_complete', backupSuccess ? this.backups.map(b => b.toJSON()) : false)
+  }
+
+  async requestApplyBackup(socket, id) {
+    // Only Root User allowed
+    var client = socket.sheepClient
+    if (!client || !client.user) {
+      Logger.error(`[BackupManager] Invalid user attempting to create backup`)
+      socket.emit('apply_backup_complete', false)
+      return
+    } else if (!client.user.isRoot) {
+      Logger.error(`[BackupManager] Non-Root user attempting to create backup`)
+      socket.emit('apply_backup_complete', false)
+      return
+    }
+
+    var backup = this.backups.find(b => b.id === id)
+    if (!backup) {
+      socket.emit('apply_backup_complete', false)
+      return
+    }
+    const zip = new StreamZip.async({ file: backup.fullPath })
+    await zip.extract('config/', this.db.ConfigPath)
+    if (backup.backupMetadataCovers) {
+      var metadataBooksPath = Path.join(this.MetadataPath, 'books')
+      await zip.extract('metadata-books/', metadataBooksPath)
+    }
+    await this.db.reinit()
+    socket.emit('apply_backup_complete', true)
+    socket.broadcast.emit('backup_applied')
+  }
+
+  async setLastBackup() {
+    this.backups.sort((a, b) => b.createdAt - a.createdAt)
+    var lastBackup = this.backups.shift()
+
+    const zip = new StreamZip.async({ file: lastBackup.fullPath })
+    await zip.extract('config/', this.db.ConfigPath)
+    console.log('Set Last Backup')
+    await this.db.reinit()
+  }
+
+  async loadBackups() {
+    try {
+      var filesInDir = await fs.readdir(this.BackupPath)
+      for (let i = 0; i < filesInDir.length; i++) {
+        var filename = filesInDir[i]
+        if (filename.endsWith('.audiobookshelf')) {
+          var fullFilePath = Path.join(this.BackupPath, filename)
+          const zip = new StreamZip.async({ file: fullFilePath })
+          const data = await zip.entryData('details')
+          var details = data.toString('utf8').split('\n')
+
+          var backup = new Backup({ details, fullPath: fullFilePath })
+          backup.fileSize = await getFileSize(backup.fullPath)
+          var existingBackupWithId = this.backups.find(b => b.id === backup.id)
+          if (existingBackupWithId) {
+            Logger.warn(`[BackupManager] Backup already loaded with id ${backup.id} - ignoring`)
+          } else {
+            this.backups.push(backup)
+          }
+
+
+          Logger.debug(`[BackupManager] Backup found "${backup.id}"`)
+          zip.close()
+        }
+      }
+      Logger.info(`[BackupManager] ${this.backups.length} Backups Found`)
+    } catch (error) {
+      Logger.error('[BackupManager] Failed to load backups', error)
+    }
+  }
+
+  async runBackup() {
+    Logger.info(`[BackupManager] Running Backup`)
+    var metadataBooksPath = this.serverSettings.backupMetadataCovers ? Path.join(this.MetadataPath, 'books') : null
+
+    var newBackup = new Backup()
+
+    const newBackData = {
+      backupMetadataCovers: this.serverSettings.backupMetadataCovers,
+      backupDirPath: this.BackupPath
+    }
+    newBackup.setData(newBackData)
+
+    var zipResult = await this.zipBackup(this.db.ConfigPath, metadataBooksPath, newBackup).then(() => true).catch((error) => {
+      Logger.error(`[BackupManager] Backup Failed ${error}`)
+      return false
+    })
+    if (zipResult) {
+      Logger.info(`[BackupManager] Backup successful ${newBackup.id}`)
+      await filePerms(newBackup.fullPath, 0o774, this.Uid, this.Gid)
+      newBackup.fileSize = await getFileSize(newBackup.fullPath)
+      var existingIndex = this.backups.findIndex(b => b.id === newBackup.id)
+      if (existingIndex >= 0) {
+        this.backups.splice(existingIndex, 1, newBackup)
+      } else {
+        this.backups.push(newBackup)
+      }
+
+      // Check remove oldest backup
+      if (this.backups.length > this.serverSettings.backupsToKeep) {
+        this.backups.sort((a, b) => a.createdAt - b.createdAt)
+
+        var oldBackup = this.backups.shift()
+        Logger.debug(`[BackupManager] Removing old backup ${oldBackup.id}`)
+        this.removeBackup(oldBackup)
+      }
+      return true
+    } else {
+      return false
+    }
+  }
+
+  async removeBackup(backup) {
+    try {
+      await fs.remove(backup.fullPath)
+      this.backups = this.backups.filter(b => b.id !== backup.id)
+      Logger.info(`[BackupManager] Backup "${backup.id}" Removed`)
+    } catch (error) {
+      Logger.error(`[BackupManager] Failed to remove backup`, error)
+    }
+  }
+
+  zipBackup(configPath, metadataBooksPath, backup) {
+    return new Promise((resolve, reject) => {
+      // create a file to stream archive data to
+      const output = fs.createWriteStream(backup.fullPath)
+      const archive = archiver('zip', {
+        zlib: { level: 9 } // Sets the compression level.
+      })
+
+      // listen for all archive data to be written
+      // 'close' event is fired only when a file descriptor is involved
+      output.on('close', () => {
+        Logger.info('[BackupManager]', archive.pointer() + ' total bytes')
+        resolve()
+      })
+
+      // This event is fired when the data source is drained no matter what was the data source.
+      // It is not part of this library but rather from the NodeJS Stream API.
+      // @see: https://nodejs.org/api/stream.html#stream_event_end
+      output.on('end', () => {
+        Logger.debug('Data has been drained')
+      })
+
+      // good practice to catch warnings (ie stat failures and other non-blocking errors)
+      archive.on('warning', function (err) {
+        if (err.code === 'ENOENT') {
+          // log warning
+          Logger.warn(`[BackupManager] Archiver warning: ${err.message}`)
+        } else {
+          // throw error
+          Logger.error(`[BackupManager] Archiver error: ${err.message}`)
+          // throw err
+          reject(err)
+        }
+      })
+      archive.on('error', function (err) {
+        Logger.error(`[BackupManager] Archiver error: ${err.message}`)
+        reject(err)
+      })
+
+      // pipe archive data to the file
+      archive.pipe(output)
+
+      archive.directory(configPath, 'config')
+      if (metadataBooksPath) {
+        Logger.debug(`[BackupManager] Backing up Metadata Books "${metadataBooksPath}"`)
+        archive.directory(metadataBooksPath, 'metadata-books')
+      }
+      archive.append(backup.detailsString, { name: 'details' })
+
+      archive.finalize()
+    })
+  }
+}
+module.exports = BackupManager

server/Db.js

@@ -70,6 +70,14 @@ class Db {
     return defaultLibrary
   }
 
+  reinit() {
+    this.audiobooksDb = new njodb.Database(this.AudiobooksPath)
+    this.usersDb = new njodb.Database(this.UsersPath)
+    this.librariesDb = new njodb.Database(this.LibrariesPath, { datastores: 2 })
+    this.settingsDb = new njodb.Database(this.SettingsPath, { datastores: 2 })
+    return this.init()
+  }
+
   async init() {
     await this.load()
 

server/Server.js

@@ -18,6 +18,7 @@ const Auth = require('./Auth')
 const Watcher = require('./Watcher')
 const Scanner = require('./Scanner')
 const Db = require('./Db')
+const BackupManager = require('./BackupManager')
 const ApiController = require('./ApiController')
 const HlsController = require('./HlsController')
 const StreamManager = require('./StreamManager')
@@ -25,7 +26,6 @@ const RssFeeds = require('./RssFeeds')
 const DownloadManager = require('./DownloadManager')
 const CoverController = require('./CoverController')
 
-
 class Server {
   constructor(PORT, UID, GID, CONFIG_PATH, METADATA_PATH, AUDIOBOOK_PATH) {
     this.Port = PORT
@@ -42,13 +42,14 @@ class Server {
 
     this.db = new Db(this.ConfigPath, this.AudiobookPath)
     this.auth = new Auth(this.db)
+    this.backupManager = new BackupManager(this.MetadataPath, this.Uid, this.Gid, this.db)
     this.watcher = new Watcher(this.AudiobookPath)
     this.coverController = new CoverController(this.db, this.MetadataPath, this.AudiobookPath)
     this.scanner = new Scanner(this.AudiobookPath, this.MetadataPath, this.db, this.coverController, this.emitter.bind(this))
     this.streamManager = new StreamManager(this.db, this.MetadataPath)
     this.rssFeeds = new RssFeeds(this.Port, this.db)
     this.downloadManager = new DownloadManager(this.db, this.MetadataPath, this.AudiobookPath, this.emitter.bind(this))
-    this.apiController = new ApiController(this.MetadataPath, this.db, this.scanner, this.auth, this.streamManager, this.rssFeeds, this.downloadManager, this.coverController, this.watcher, this.emitter.bind(this), this.clientEmitter.bind(this))
+    this.apiController = new ApiController(this.MetadataPath, this.db, this.scanner, this.auth, this.streamManager, this.rssFeeds, this.downloadManager, this.coverController, this.backupManager, this.watcher, this.emitter.bind(this), this.clientEmitter.bind(this))
     this.hlsController = new HlsController(this.db, this.scanner, this.auth, this.streamManager, this.emitter.bind(this), this.streamManager.StreamsPath)
 
     this.expressApp = null
@@ -101,6 +102,7 @@ class Server {
     this.auth.init()
 
     await this.purgeMetadata()
+    await this.backupManager.init()
 
     this.watcher.initWatcher(this.libraries)
     this.watcher.on('files', this.filesChanged.bind(this))
@@ -158,6 +160,18 @@ class Server {
       res.sendFile(fullPath)
     })
 
+    // EBook static file routes
+    app.get('/ebook/:library/:folder/*', (req, res) => {
+      var library = this.libraries.find(lib => lib.id === req.params.library)
+      if (!library) return res.sendStatus(404)
+      var folder = library.folders.find(fol => fol.id === req.params.folder)
+      if (!folder) return res.status(404).send('Folder not found')
+
+      var remainingPath = req.params['0']
+      var fullPath = Path.join(folder.fullPath, remainingPath)
+      res.sendFile(fullPath)
+    })
+
     // Client routes
     app.get('/audiobook/:id', (req, res) => res.sendFile(Path.join(distPath, 'index.html')))
     app.get('/audiobook/:id/edit', (req, res) => res.sendFile(Path.join(distPath, 'index.html')))
@@ -235,8 +249,13 @@ class Server {
       socket.on('download', (payload) => this.downloadManager.downloadSocketRequest(socket, payload))
       socket.on('remove_download', (downloadId) => this.downloadManager.removeSocketRequest(socket, downloadId))
 
+      // Logs
       socket.on('set_log_listener', (level) => Logger.addSocketListener(socket, level))
 
+      // Backups
+      socket.on('create_backup', () => this.backupManager.requestCreateBackup(socket))
+      socket.on('apply_backup', (id) => this.backupManager.requestApplyBackup(socket, id))
+
       socket.on('test', () => {
         socket.emit('test_received', socket.id)
       })
@@ -448,7 +467,8 @@ class Server {
       configPath: this.ConfigPath,
       user: client.user.toJSONForBrowser(),
       stream: client.stream || null,
-      librariesScanning: this.scanner.librariesScanning
+      librariesScanning: this.scanner.librariesScanning,
+      backups: (this.backupManager.backups || []).map(b => b.toJSON())
     }
     client.socket.emit('init', initialPayload)
 

server/StreamManager.js

@@ -65,7 +65,7 @@ class StreamManager {
       if (!dirs || !dirs.length) return true
 
       await Promise.all(dirs.map(async (dirname) => {
-        if (dirname !== 'streams' && dirname !== 'books' && dirname !== 'downloads') {
+        if (dirname !== 'streams' && dirname !== 'books' && dirname !== 'downloads' && dirname !== 'backups') {
           var fullPath = Path.join(this.MetadataPath, dirname)
           Logger.warn(`Removing OLD Orphan Stream ${dirname}`)
           return fs.remove(fullPath)

server/objects/Audiobook.js

@@ -223,6 +223,7 @@ class Audiobook {
       audioFiles: (this.audioFiles || []).map(audioFile => audioFile.toJSON()),
       otherFiles: (this.otherFiles || []).map(otherFile => otherFile.toJSON()),
       ebooks: (this.ebooks || []).map(ebook => ebook.toJSON()),
+      numEbooks: this.hasEpub ? 1 : 0,
       tags: this.tags,
       book: this.bookToJSON(),
       tracks: this.tracksToJSON(),

server/objects/Backup.js

@@ -0,0 +1,75 @@
+const Path = require('path')
+const date = require('date-and-time')
+
+class Backup {
+  constructor(data = null) {
+    this.id = null
+    this.datePretty = null
+    this.backupMetadataCovers = null
+
+    this.backupDirPath = null
+    this.filename = null
+    this.path = null
+    this.fullPath = null
+
+    this.fileSize = null
+    this.createdAt = null
+
+    if (data) {
+      this.construct(data)
+    }
+  }
+
+  get detailsString() {
+    var details = []
+    details.push(this.id)
+    details.push(this.backupMetadataCovers ? '1' : '0')
+    details.push(this.createdAt)
+    return details.join('\n')
+  }
+
+  construct(data) {
+    this.id = data.details[0]
+    this.backupMetadataCovers = data.details[1] === '1'
+    this.createdAt = Number(data.details[2])
+
+    this.datePretty = date.format(new Date(this.createdAt), 'ddd, MMM D YYYY HH:mm')
+
+    this.backupDirPath = Path.dirname(data.fullPath)
+    this.filename = Path.basename(data.fullPath)
+    this.path = Path.join('backups', this.filename)
+    this.fullPath = data.fullPath
+  }
+
+  toJSON() {
+    return {
+      id: this.id,
+      backupMetadataCovers: this.backupMetadataCovers,
+      backupDirPath: this.backupDirPath,
+      datePretty: this.datePretty,
+      path: this.path,
+      fullPath: this.fullPath,
+      path: this.path,
+      filename: this.filename,
+      fileSize: this.fileSize,
+      createdAt: this.createdAt
+    }
+  }
+
+  setData(data) {
+    this.id = date.format(new Date(), 'YYYY-MM-DD[T]HHmm')
+    this.datePretty = date.format(new Date(), 'ddd, MMM D YYYY HH:mm')
+
+    this.backupMetadataCovers = data.backupMetadataCovers
+
+    this.backupDirPath = data.backupDirPath
+
+    this.filename = this.id + '.audiobookshelf'
+    this.path = Path.join('backups', this.filename)
+    this.fullPath = Path.join(this.backupDirPath, this.filename)
+    console.log('Backup fullpath', this.fullPath)
+
+    this.createdAt = Date.now()
+  }
+}
+module.exports = Backup

server/objects/ServerSettings.js

@@ -5,14 +5,28 @@ class ServerSettings {
   constructor(settings) {
     this.id = 'server-settings'
 
+    // Misc/Unused
     this.autoTagNew = false
     this.newTagExpireDays = 15
+
+    // Scanner
     this.scannerParseSubtitle = false
     this.scannerFindCovers = false
+
+    // Metadata
     this.coverDestination = CoverDestination.METADATA
     this.saveMetadataFile = false
+
+    // Security/Rate limits
     this.rateLimitLoginRequests = 10
     this.rateLimitLoginWindow = 10 * 60 * 1000 // 10 Minutes
+
+    // Backups
+    // this.backupSchedule = '0 1 * * *' // If false then auto-backups are disabled (default every day at 1am)
+    this.backupSchedule = false
+    this.backupsToKeep = 2
+    this.backupMetadataCovers = true
+
     this.logLevel = Logger.logLevel
 
     if (settings) {
@@ -29,6 +43,11 @@ class ServerSettings {
     this.saveMetadataFile = !!settings.saveMetadataFile
     this.rateLimitLoginRequests = !isNaN(settings.rateLimitLoginRequests) ? Number(settings.rateLimitLoginRequests) : 10
     this.rateLimitLoginWindow = !isNaN(settings.rateLimitLoginWindow) ? Number(settings.rateLimitLoginWindow) : 10 * 60 * 1000 // 10 Minutes
+
+    this.backupSchedule = settings.backupSchedule || false
+    this.backupsToKeep = settings.backupsToKeep || 2
+    this.backupMetadataCovers = settings.backupMetadataCovers !== false
+
     this.logLevel = settings.logLevel || Logger.logLevel
 
     if (this.logLevel !== Logger.logLevel) {
@@ -47,6 +66,9 @@ class ServerSettings {
       saveMetadataFile: !!this.saveMetadataFile,
       rateLimitLoginRequests: this.rateLimitLoginRequests,
       rateLimitLoginWindow: this.rateLimitLoginWindow,
+      backupSchedule: this.backupSchedule,
+      backupsToKeep: this.backupsToKeep,
+      backupMetadataCovers: this.backupMetadataCovers,
       logLevel: this.logLevel
     }
   }