Merge branch 'refs/heads/master' into mf/rssInboundManager

# Conflicts:
#	client/components/widgets/PodcastDetailsEdit.vue
#	client/pages/config/rss-feeds.vue
#	client/strings/en-us.json
#	server/controllers/PodcastController.js
#	server/models/Podcast.js
#	server/routers/ApiRouter.js
#	server/utils/queries/libraryItemsPodcastFilters.js

server/utils/comicBookExtractors.js

@@ -0,0 +1,207 @@
+const Path = require('path')
+const os = require('os')
+const unrar = require('node-unrar-js')
+const Logger = require('../Logger')
+const fs = require('../libs/fsExtra')
+const StreamZip = require('../libs/nodeStreamZip')
+const Archive = require('../libs/libarchive/archive')
+const { isWritable } = require('./fileUtils')
+
+class AbstractComicBookExtractor {
+  constructor(comicPath) {
+    this.comicPath = comicPath
+  }
+
+  async getBuffer() {
+    if (!(await fs.pathExists(this.comicPath))) {
+      Logger.error(`[parseComicMetadata] Comic path does not exist "${this.comicPath}"`)
+      return null
+    }
+    try {
+      return fs.readFile(this.comicPath)
+    } catch (error) {
+      Logger.error(`[parseComicMetadata] Failed to read comic at "${this.comicPath}"`, error)
+      return null
+    }
+  }
+
+  async open() {
+    throw new Error('Not implemented')
+  }
+
+  async getFilePaths() {
+    throw new Error('Not implemented')
+  }
+
+  async extractToFile(filePath, outputFilePath) {
+    throw new Error('Not implemented')
+  }
+
+  async extractToBuffer(filePath) {
+    throw new Error('Not implemented')
+  }
+
+  close() {
+    throw new Error('Not implemented')
+  }
+}
+
+class CbrComicBookExtractor extends AbstractComicBookExtractor {
+  constructor(comicPath) {
+    super(comicPath)
+    this.archive = null
+    this.tmpDir = null
+  }
+
+  async open() {
+    this.tmpDir = global.MetadataPath ? Path.join(global.MetadataPath, 'tmp') : os.tmpdir()
+    await fs.ensureDir(this.tmpDir)
+    if (!(await isWritable(this.tmpDir))) throw new Error(`[CbrComicBookExtractor] Temp directory "${this.tmpDir}" is not writable`)
+    this.archive = await unrar.createExtractorFromFile({ filepath: this.comicPath, targetPath: this.tmpDir })
+    Logger.debug(`[CbrComicBookExtractor] Opened comic book "${this.comicPath}". Using temp directory "${this.tmpDir}" for extraction.`)
+  }
+
+  async getFilePaths() {
+    if (!this.archive) return null
+    const list = this.archive.getFileList()
+    const fileHeaders = [...list.fileHeaders]
+    const filePaths = fileHeaders.filter((fh) => !fh.flags.directory).map((fh) => fh.name)
+    Logger.debug(`[CbrComicBookExtractor] Found ${filePaths.length} files in comic book "${this.comicPath}"`)
+    return filePaths
+  }
+
+  async removeEmptyParentDirs(file) {
+    let dir = Path.dirname(file)
+    while (dir !== '.') {
+      const fullDirPath = Path.join(this.tmpDir, dir)
+      const files = await fs.readdir(fullDirPath)
+      if (files.length > 0) break
+      await fs.remove(fullDirPath)
+      dir = Path.dirname(dir)
+    }
+  }
+
+  async extractToBuffer(file) {
+    if (!this.archive) return null
+    const extracted = this.archive.extract({ files: [file] })
+    const files = [...extracted.files]
+    const filePath = Path.join(this.tmpDir, files[0].fileHeader.name)
+    const fileData = await fs.readFile(filePath)
+    await fs.remove(filePath)
+    await this.removeEmptyParentDirs(files[0].fileHeader.name)
+    Logger.debug(`[CbrComicBookExtractor] Extracted file "${file}" from comic book "${this.comicPath}" to buffer, size: ${fileData.length}`)
+    return fileData
+  }
+
+  async extractToFile(file, outputFilePath) {
+    if (!this.archive) return false
+    const extracted = this.archive.extract({ files: [file] })
+    const files = [...extracted.files]
+    const extractedFilePath = Path.join(this.tmpDir, files[0].fileHeader.name)
+    await fs.move(extractedFilePath, outputFilePath, { overwrite: true })
+    await this.removeEmptyParentDirs(files[0].fileHeader.name)
+    Logger.debug(`[CbrComicBookExtractor] Extracted file "${file}" from comic book "${this.comicPath}" to "${outputFilePath}"`)
+    return true
+  }
+
+  close() {
+    Logger.debug(`[CbrComicBookExtractor] Closed comic book "${this.comicPath}"`)
+  }
+}
+
+class CbzComicBookExtractor extends AbstractComicBookExtractor {
+  constructor(comicPath) {
+    super(comicPath)
+    this.archive = null
+  }
+
+  async open() {
+    const buffer = await this.getBuffer()
+    this.archive = await Archive.open(buffer)
+    Logger.debug(`[CbzComicBookExtractor] Opened comic book "${this.comicPath}"`)
+  }
+
+  async getFilePaths() {
+    if (!this.archive) return null
+    const list = await this.archive.getFilesArray()
+    const fileNames = list.map((fo) => fo.file._path)
+    Logger.debug(`[CbzComicBookExtractor] Found ${fileNames.length} files in comic book "${this.comicPath}"`)
+    return fileNames
+  }
+
+  async extractToBuffer(file) {
+    if (!this.archive) return null
+    const extracted = await this.archive.extractSingleFile(file)
+    Logger.debug(`[CbzComicBookExtractor] Extracted file "${file}" from comic book "${this.comicPath}" to buffer, size: ${extracted?.fileData.length}`)
+    return extracted?.fileData
+  }
+
+  async extractToFile(file, outputFilePath) {
+    const data = await this.extractToBuffer(file)
+    if (!data) return false
+    await fs.writeFile(outputFilePath, data)
+    Logger.debug(`[CbzComicBookExtractor] Extracted file "${file}" from comic book "${this.comicPath}" to "${outputFilePath}"`)
+    return true
+  }
+
+  close() {
+    this.archive?.close()
+    Logger.debug(`[CbzComicBookExtractor] Closed comic book "${this.comicPath}"`)
+  }
+}
+
+class CbzStreamZipComicBookExtractor extends AbstractComicBookExtractor {
+  constructor(comicPath) {
+    super(comicPath)
+    this.archive = null
+  }
+
+  async open() {
+    this.archive = new StreamZip.async({ file: this.comicPath })
+    Logger.debug(`[CbzStreamZipComicBookExtractor] Opened comic book "${this.comicPath}"`)
+  }
+
+  async getFilePaths() {
+    if (!this.archive) return null
+    const entries = await this.archive.entries()
+    const fileNames = Object.keys(entries).filter((entry) => !entries[entry].isDirectory)
+    Logger.debug(`[CbzStreamZipComicBookExtractor] Found ${fileNames.length} files in comic book "${this.comicPath}"`)
+    return fileNames
+  }
+
+  async extractToBuffer(file) {
+    if (!this.archive) return null
+    const extracted = await this.archive?.entryData(file)
+    Logger.debug(`[CbzStreamZipComicBookExtractor] Extracted file "${file}" from comic book "${this.comicPath}" to buffer, size: ${extracted.length}`)
+    return extracted
+  }
+
+  async extractToFile(file, outputFilePath) {
+    if (!this.archive) return false
+    try {
+      await this.archive.extract(file, outputFilePath)
+      Logger.debug(`[CbzStreamZipComicBookExtractor] Extracted file "${file}" from comic book "${this.comicPath}" to "${outputFilePath}"`)
+      return true
+    } catch (error) {
+      Logger.error(`[CbzStreamZipComicBookExtractor] Failed to extract file "${file}" to "${outputFilePath}"`, error)
+      return false
+    }
+  }
+
+  close() {
+    this.archive?.close()
+    Logger.debug(`[CbzStreamZipComicBookExtractor] Closed comic book "${this.comicPath}"`)
+  }
+}
+
+function createComicBookExtractor(comicPath) {
+  const ext = Path.extname(comicPath).toLowerCase()
+  if (ext === '.cbr') {
+    return new CbrComicBookExtractor(comicPath)
+  } else if (ext === '.cbz') {
+    return new CbzStreamZipComicBookExtractor(comicPath)
+  } else {
+    throw new Error(`Unsupported comic book format "${ext}"`)
+  }
+}
+module.exports = { createComicBookExtractor }

server/utils/constants.js

@@ -49,9 +49,7 @@ module.exports.AudioMimeType = {
   WEBMA: 'audio/webm',
   MKA: 'audio/x-matroska',
   AWB: 'audio/amr-wb',
-  CAF: 'audio/x-caf'
+  CAF: 'audio/x-caf',
+  MPEG: 'audio/mpeg',
+  MPG: 'audio/mpeg'
 }
-
-module.exports.VideoMimeType = {
-  MP4: 'video/mp4'
-}

server/utils/downloadWorker.js

@@ -1,92 +0,0 @@
-const Ffmpeg = require('../libs/fluentFfmpeg')
-
-if (process.env.FFMPEG_PATH) {
-  Ffmpeg.setFfmpegPath(process.env.FFMPEG_PATH)
-}
-
-const { parentPort, workerData } = require("worker_threads")
-
-parentPort.postMessage({
-  type: 'FFMPEG',
-  level: 'debug',
-  log: '[DownloadWorker] Starting Worker...'
-})
-
-const ffmpegCommand = Ffmpeg()
-const startTime = Date.now()
-
-workerData.inputs.forEach((inputData) => {
-  ffmpegCommand.input(inputData.input)
-  if (inputData.options) ffmpegCommand.inputOption(inputData.options)
-})
-
-if (workerData.options) ffmpegCommand.addOption(workerData.options)
-if (workerData.outputOptions && workerData.outputOptions.length) ffmpegCommand.addOutputOption(workerData.outputOptions)
-ffmpegCommand.output(workerData.output)
-
-var isKilled = false
-
-async function runFfmpeg() {
-  var success = await new Promise((resolve) => {
-    ffmpegCommand.on('start', (command) => {
-      parentPort.postMessage({
-        type: 'FFMPEG',
-        level: 'info',
-        log: '[DownloadWorker] FFMPEG concat started with command: ' + command
-      })
-    })
-
-    ffmpegCommand.on('stderr', (stdErrline) => {
-      parentPort.postMessage({
-        type: 'FFMPEG',
-        level: 'debug',
-        log: '[DownloadWorker] Ffmpeg Stderr: ' + stdErrline
-      })
-    })
-
-    ffmpegCommand.on('error', (err, stdout, stderr) => {
-      if (err.message && err.message.includes('SIGKILL')) {
-        // This is an intentional SIGKILL
-        parentPort.postMessage({
-          type: 'FFMPEG',
-          level: 'info',
-          log: '[DownloadWorker] User Killed worker'
-        })
-      } else {
-        parentPort.postMessage({
-          type: 'FFMPEG',
-          level: 'error',
-          log: '[DownloadWorker] Ffmpeg Err: ' + err.message
-        })
-      }
-      resolve(false)
-    })
-
-    ffmpegCommand.on('end', (stdout, stderr) => {
-      parentPort.postMessage({
-        type: 'FFMPEG',
-        level: 'info',
-        log: '[DownloadWorker] worker ended'
-      })
-      resolve(true)
-    })
-    ffmpegCommand.run()
-  })
-
-  var resultMessage = {
-    type: 'RESULT',
-    isKilled,
-    elapsed: Date.now() - startTime,
-    success
-  }
-  parentPort.postMessage(resultMessage)
-}
-
-parentPort.on('message', (message) => {
-  if (message === 'STOP') {
-    isKilled = true
-    ffmpegCommand.kill()
-  }
-})
-
-runFfmpeg()

server/utils/ffmpegHelpers.js

@@ -1,13 +1,15 @@
 const axios = require('axios')
 const Ffmpeg = require('../libs/fluentFfmpeg')
+const ffmpgegUtils = require('../libs/fluentFfmpeg/utils')
 const fs = require('../libs/fsExtra')
 const Path = require('path')
 const Logger = require('../Logger')
-const { filePathToPOSIX } = require('./fileUtils')
+const { filePathToPOSIX, copyToExisting } = require('./fileUtils')
+const LibraryItem = require('../objects/LibraryItem')
 
 function escapeSingleQuotes(path) {
   // return path.replace(/'/g, '\'\\\'\'')
-  return filePathToPOSIX(path).replace(/ /g, '\\ ').replace(/'/g, '\\\'')
+  return filePathToPOSIX(path).replace(/ /g, '\\ ').replace(/'/g, "\\'")
 }
 
 // Returns first track start time
@@ -19,7 +21,7 @@ async function writeConcatFile(tracks, outputPath, startTime = 0) {
   // Find first track greater than startTime
   if (startTime > 0) {
     var currTrackEnd = 0
-    var startingTrack = tracks.find(t => {
+    var startingTrack = tracks.find((t) => {
       currTrackEnd += t.duration
       return startTime < currTrackEnd
     })
@@ -29,8 +31,8 @@ async function writeConcatFile(tracks, outputPath, startTime = 0) {
     }
   }
 
-  var tracksToInclude = tracks.filter(t => t.index >= trackToStartWithIndex)
-  var trackPaths = tracksToInclude.map(t => {
+  var tracksToInclude = tracks.filter((t) => t.index >= trackToStartWithIndex)
+  var trackPaths = tracksToInclude.map((t) => {
     var line = 'file ' + escapeSingleQuotes(t.metadata.path) + '\n' + `duration ${t.duration}`
     return line
   })
@@ -51,8 +53,9 @@ async function extractCoverArt(filepath, outputpath) {
   await fs.ensureDir(dirname)
 
   return new Promise((resolve) => {
+    /** @type {import('../libs/fluentFfmpeg/index').FfmpegCommand} */
     var ffmpeg = Ffmpeg(filepath)
-    ffmpeg.addOption(['-map 0:v', '-frames:v 1'])
+    ffmpeg.addOption(['-map 0:v:0', '-frames:v 1'])
     ffmpeg.output(outputpath)
 
     ffmpeg.on('start', (cmd) => {
@@ -74,6 +77,7 @@ module.exports.extractCoverArt = extractCoverArt
 //This should convert based on the output file extension as well
 async function resizeImage(filePath, outputPath, width, height) {
   return new Promise((resolve) => {
+    /** @type {import('../libs/fluentFfmpeg/index').FfmpegCommand} */
     var ffmpeg = Ffmpeg(filePath)
     ffmpeg.addOption(['-vf', `scale=${width || -1}:${height || -1}`])
     ffmpeg.addOutput(outputPath)
@@ -99,6 +103,9 @@ module.exports.downloadPodcastEpisode = (podcastEpisodeDownload) => {
       url: podcastEpisodeDownload.url,
       method: 'GET',
       responseType: 'stream',
+      headers: {
+        'User-Agent': 'audiobookshelf (+https://audiobookshelf.org)'
+      },
       timeout: 30000
     }).catch((error) => {
       Logger.error(`[ffmpegHelpers] Failed to download podcast episode with url "${podcastEpisodeDownload.url}"`, error)
@@ -106,37 +113,34 @@ module.exports.downloadPodcastEpisode = (podcastEpisodeDownload) => {
     })
     if (!response) return resolve(false)
 
+    /** @type {import('../libs/fluentFfmpeg/index').FfmpegCommand} */
     const ffmpeg = Ffmpeg(response.data)
     ffmpeg.addOption('-loglevel debug') // Debug logs printed on error
-    ffmpeg.outputOptions(
-      '-c:a', 'copy',
-      '-map', '0:a',
-      '-metadata', 'podcast=1'
-    )
+    ffmpeg.outputOptions('-c:a', 'copy', '-map', '0:a', '-metadata', 'podcast=1')
 
     const podcastMetadata = podcastEpisodeDownload.libraryItem.media.metadata
     const podcastEpisode = podcastEpisodeDownload.podcastEpisode
     const finalSizeInBytes = Number(podcastEpisode.enclosure?.length || 0)
 
     const taggings = {
-      'album': podcastMetadata.title,
+      album: podcastMetadata.title,
       'album-sort': podcastMetadata.title,
-      'artist': podcastMetadata.author,
+      artist: podcastMetadata.author,
       'artist-sort': podcastMetadata.author,
-      'comment': podcastEpisode.description,
-      'subtitle': podcastEpisode.subtitle,
-      'disc': podcastEpisode.season,
-      'genre': podcastMetadata.genres.length ? podcastMetadata.genres.join(';') : null,
-      'language': podcastMetadata.language,
-      'MVNM': podcastMetadata.title,
-      'MVIN': podcastEpisode.episode,
-      'track': podcastEpisode.episode,
+      comment: podcastEpisode.description,
+      subtitle: podcastEpisode.subtitle,
+      disc: podcastEpisode.season,
+      genre: podcastMetadata.genres.length ? podcastMetadata.genres.join(';') : null,
+      language: podcastMetadata.language,
+      MVNM: podcastMetadata.title,
+      MVIN: podcastEpisode.episode,
+      track: podcastEpisode.episode,
       'series-part': podcastEpisode.episode,
-      'title': podcastEpisode.title,
+      title: podcastEpisode.title,
       'title-sort': podcastEpisode.title,
-      'year': podcastEpisode.pubYear,
-      'date': podcastEpisode.pubDate,
-      'releasedate': podcastEpisode.pubDate,
+      year: podcastEpisode.pubYear,
+      date: podcastEpisode.pubDate,
+      releasedate: podcastEpisode.pubDate,
       'itunes-id': podcastMetadata.itunesId,
       'podcast-type': podcastMetadata.type,
       'episode-type': podcastMetadata.episodeType
@@ -185,3 +189,295 @@ module.exports.downloadPodcastEpisode = (podcastEpisodeDownload) => {
     ffmpeg.run()
   })
 }
+
+/**
+ * Generates ffmetadata file content from the provided metadata object and chapters array.
+ * @param {Object} metadata - The input metadata object.
+ * @param {Array|null} chapters - An array of chapter objects.
+ * @returns {string} - The ffmetadata file content.
+ */
+function generateFFMetadata(metadata, chapters) {
+  let ffmetadataContent = ';FFMETADATA1\n'
+
+  // Add global metadata
+  for (const key in metadata) {
+    if (metadata[key]) {
+      ffmetadataContent += `${key}=${escapeFFMetadataValue(metadata[key])}\n`
+    }
+  }
+
+  // Add chapters
+  if (chapters) {
+    chapters.forEach((chapter) => {
+      ffmetadataContent += '\n[CHAPTER]\n'
+      ffmetadataContent += `TIMEBASE=1/1000\n`
+      ffmetadataContent += `START=${Math.floor(chapter.start * 1000)}\n`
+      ffmetadataContent += `END=${Math.floor(chapter.end * 1000)}\n`
+      if (chapter.title) {
+        ffmetadataContent += `title=${escapeFFMetadataValue(chapter.title)}\n`
+      }
+    })
+  }
+
+  return ffmetadataContent
+}
+
+module.exports.generateFFMetadata = generateFFMetadata
+
+/**
+ * Writes FFmpeg metadata file with the given metadata and chapters.
+ *
+ * @param {Object} metadata - The metadata object.
+ * @param {Array} chapters - The array of chapter objects.
+ * @param {string} ffmetadataPath - The path to the FFmpeg metadata file.
+ * @returns {Promise<boolean>} - A promise that resolves to true if the file was written successfully, false otherwise.
+ */
+async function writeFFMetadataFile(metadata, chapters, ffmetadataPath) {
+  try {
+    await fs.writeFile(ffmetadataPath, generateFFMetadata(metadata, chapters))
+    Logger.debug(`[ffmpegHelpers] Wrote ${ffmetadataPath}`)
+    return true
+  } catch (error) {
+    Logger.error(`[ffmpegHelpers] Write ${ffmetadataPath} failed`, error)
+    return false
+  }
+}
+
+module.exports.writeFFMetadataFile = writeFFMetadataFile
+
+/**
+ * Adds an ffmetadata and optionally a cover image to an audio file using fluent-ffmpeg.
+ *
+ * @param {string} audioFilePath - Path to the input audio file.
+ * @param {string|null} coverFilePath - Path to the cover image file.
+ * @param {string} metadataFilePath - Path to the ffmetadata file.
+ * @param {number} track - The track number to embed in the audio file.
+ * @param {string} mimeType - The MIME type of the audio file.
+ * @param {function(number): void|null} progressCB - A callback function to report progress.
+ * @param {import('../libs/fluentFfmpeg/index').FfmpegCommand} ffmpeg - The Ffmpeg instance to use (optional). Used for dependency injection in tests.
+ * @param {function(string, string): Promise<void>} copyFunc - The function to use for copying files (optional). Used for dependency injection in tests.
+ * @returns {Promise<void>} A promise that resolves if the operation is successful, rejects otherwise.
+ */
+async function addCoverAndMetadataToFile(audioFilePath, coverFilePath, metadataFilePath, track, mimeType, progressCB = null, ffmpeg = Ffmpeg(), copyFunc = copyToExisting) {
+  const isMp4 = mimeType === 'audio/mp4'
+  const isMp3 = mimeType === 'audio/mpeg'
+
+  const audioFileDir = Path.dirname(audioFilePath)
+  const audioFileExt = Path.extname(audioFilePath)
+  const audioFileBaseName = Path.basename(audioFilePath, audioFileExt)
+  const tempFilePath = filePathToPOSIX(Path.join(audioFileDir, `${audioFileBaseName}.tmp${audioFileExt}`))
+
+  return new Promise((resolve, reject) => {
+    ffmpeg.input(audioFilePath).input(metadataFilePath).outputOptions([
+      '-map 0:a', // map audio stream from input file
+      '-map_metadata 1', // map metadata tags from metadata file first
+      '-map_metadata 0', // add additional metadata tags from input file
+      '-map_chapters 1', // map chapters from metadata file
+      '-c copy' // copy streams
+    ])
+
+    if (track && !isNaN(track)) {
+      ffmpeg.outputOptions(['-metadata track=' + track])
+    }
+
+    if (isMp4) {
+      ffmpeg.outputOptions([
+        '-f mp4' // force output format to mp4
+      ])
+    } else if (isMp3) {
+      ffmpeg.outputOptions([
+        '-id3v2_version 3' // set ID3v2 version to 3
+      ])
+    }
+
+    if (coverFilePath) {
+      ffmpeg.input(coverFilePath).outputOptions([
+        '-map 2:v', // map video stream from cover image file
+        '-disposition:v:0 attached_pic', // set cover image as attached picture
+        '-metadata:s:v',
+        'title=Cover', // add title metadata to cover image stream
+        '-metadata:s:v',
+        'comment=Cover' // add comment metadata to cover image stream
+      ])
+      const ext = Path.extname(coverFilePath).toLowerCase()
+      if (ext === '.webp') {
+        ffmpeg.outputOptions([
+          '-c:v mjpeg' // convert webp images to jpeg
+        ])
+      }
+    } else {
+      ffmpeg.outputOptions([
+        '-map 0:v?' // retain video stream from input file if exists
+      ])
+    }
+
+    ffmpeg
+      .output(tempFilePath)
+      .on('start', (commandLine) => {
+        Logger.debug('[ffmpegHelpers] Spawned Ffmpeg with command: ' + commandLine)
+      })
+      .on('progress', (progress) => {
+        if (!progressCB || !progress.percent) return
+        Logger.debug(`[ffmpegHelpers] Progress: ${progress.percent}%`)
+        progressCB(progress.percent)
+      })
+      .on('end', async (stdout, stderr) => {
+        Logger.debug('[ffmpegHelpers] ffmpeg stdout:', stdout)
+        Logger.debug('[ffmpegHelpers] ffmpeg stderr:', stderr)
+        Logger.debug('[ffmpegHelpers] Moving temp file to audio file path:', `"${tempFilePath}"`, '->', `"${audioFilePath}"`)
+        try {
+          await copyFunc(tempFilePath, audioFilePath)
+          await fs.remove(tempFilePath)
+          resolve()
+        } catch (error) {
+          Logger.error(`[ffmpegHelpers] Failed to move temp file to audio file path: "${tempFilePath}" -> "${audioFilePath}"`, error)
+          reject(error)
+        }
+      })
+      .on('error', (err, stdout, stderr) => {
+        if (err.message && err.message.includes('SIGKILL')) {
+          Logger.info(`[ffmpegHelpers] addCoverAndMetadataToFile Killed by User`)
+          reject(new Error('FFMPEG_CANCELED'))
+        } else {
+          Logger.error('Error adding cover image and metadata:', err)
+          Logger.error('ffmpeg stdout:', stdout)
+          Logger.error('ffmpeg stderr:', stderr)
+          reject(err)
+        }
+      })
+
+    ffmpeg.run()
+  })
+}
+
+module.exports.addCoverAndMetadataToFile = addCoverAndMetadataToFile
+
+function escapeFFMetadataValue(value) {
+  return value.replace(/([;=\n\\#])/g, '\\$1')
+}
+
+/**
+ * Retrieves the FFmpeg metadata object for a given library item.
+ *
+ * @param {LibraryItem} libraryItem - The library item containing the media metadata.
+ * @param {number} audioFilesLength - The length of the audio files.
+ * @returns {Object} - The FFmpeg metadata object.
+ */
+function getFFMetadataObject(libraryItem, audioFilesLength) {
+  const metadata = libraryItem.media.metadata
+
+  const ffmetadata = {
+    title: metadata.title,
+    artist: metadata.authorName,
+    album_artist: metadata.authorName,
+    album: (metadata.title || '') + (metadata.subtitle ? `: ${metadata.subtitle}` : ''),
+    TIT3: metadata.subtitle, // mp3 only
+    genre: metadata.genres?.join('; '),
+    date: metadata.publishedYear,
+    comment: metadata.description,
+    description: metadata.description,
+    composer: metadata.narratorName,
+    copyright: metadata.publisher,
+    publisher: metadata.publisher, // mp3 only
+    TRACKTOTAL: `${audioFilesLength}`, // mp3 only
+    grouping: metadata.series?.map((s) => s.name + (s.sequence ? ` #${s.sequence}` : '')).join('; ')
+  }
+  Object.keys(ffmetadata).forEach((key) => {
+    if (!ffmetadata[key]) {
+      delete ffmetadata[key]
+    }
+  })
+
+  return ffmetadata
+}
+
+module.exports.getFFMetadataObject = getFFMetadataObject
+
+/**
+ * Merges audio files into a single output file using FFmpeg.
+ *
+ * @param {Array} audioTracks - The audio tracks to merge.
+ * @param {number} duration - The total duration of the audio tracks.
+ * @param {string} itemCachePath - The path to the item cache.
+ * @param {string} outputFilePath - The path to the output file.
+ * @param {import('../managers/AbMergeManager').AbMergeEncodeOptions} encodingOptions - The options for encoding the audio.
+ * @param {Function} [progressCB=null] - The callback function to track the progress of the merge.
+ * @param {import('../libs/fluentFfmpeg/index').FfmpegCommand} [ffmpeg=Ffmpeg()] - The FFmpeg instance to use for merging.
+ * @returns {Promise<void>} A promise that resolves when the audio files are merged successfully.
+ */
+async function mergeAudioFiles(audioTracks, duration, itemCachePath, outputFilePath, encodingOptions, progressCB = null, ffmpeg = Ffmpeg()) {
+  const audioBitrate = encodingOptions.bitrate || '128k'
+  const audioCodec = encodingOptions.codec || 'aac'
+  const audioChannels = encodingOptions.channels || 2
+
+  // TODO: Updated in 2.2.11 to always encode even if merging multiple m4b. This is because just using the file extension as was being done before is not enough. This can be an option or do more to check if a concat is possible.
+  // const audioRequiresEncode = audioTracks[0].metadata.ext !== '.m4b'
+  const audioRequiresEncode = true
+
+  const firstTrackIsM4b = audioTracks[0].metadata.ext.toLowerCase() === '.m4b'
+  const isOneTrack = audioTracks.length === 1
+
+  let concatFilePath = null
+  if (!isOneTrack) {
+    concatFilePath = Path.join(itemCachePath, 'files.txt')
+    if ((await writeConcatFile(audioTracks, concatFilePath)) == null) {
+      throw new Error('Failed to write concat file')
+    }
+    ffmpeg.input(concatFilePath).inputOptions(['-safe 0', '-f concat'])
+  } else {
+    ffmpeg.input(audioTracks[0].metadata.path).inputOptions(firstTrackIsM4b ? ['-f mp4'] : [])
+  }
+
+  //const logLevel = process.env.NODE_ENV === 'production' ? 'error' : 'warning'
+  ffmpeg.outputOptions(['-f mp4'])
+
+  if (audioRequiresEncode) {
+    ffmpeg.outputOptions(['-map 0:a', `-acodec ${audioCodec}`, `-ac ${audioChannels}`, `-b:a ${audioBitrate}`])
+  } else {
+    ffmpeg.outputOptions(['-max_muxing_queue_size 1000'])
+
+    if (isOneTrack && firstTrackIsM4b) {
+      ffmpeg.outputOptions(['-c copy'])
+    } else {
+      ffmpeg.outputOptions(['-c:a copy'])
+    }
+  }
+
+  ffmpeg.output(outputFilePath)
+
+  return new Promise((resolve, reject) => {
+    ffmpeg
+      .on('start', (cmd) => {
+        Logger.debug(`[ffmpegHelpers] Merge Audio Files ffmpeg command: ${cmd}`)
+      })
+      .on('progress', (progress) => {
+        if (!progressCB || !progress.timemark || !duration) return
+        // Cannot rely on progress.percent as it is not accurate for concat
+        const percent = (ffmpgegUtils.timemarkToSeconds(progress.timemark) / duration) * 100
+        progressCB(percent)
+      })
+      .on('end', async (stdout, stderr) => {
+        if (concatFilePath) await fs.remove(concatFilePath)
+        Logger.debug('[ffmpegHelpers] ffmpeg stdout:', stdout)
+        Logger.debug('[ffmpegHelpers] ffmpeg stderr:', stderr)
+        Logger.debug(`[ffmpegHelpers] Audio Files Merged Successfully`)
+        resolve()
+      })
+      .on('error', async (err, stdout, stderr) => {
+        if (concatFilePath) await fs.remove(concatFilePath)
+        if (err.message && err.message.includes('SIGKILL')) {
+          Logger.info(`[ffmpegHelpers] Merge Audio Files Killed by User`)
+          reject(new Error('FFMPEG_CANCELED'))
+        } else {
+          Logger.error(`[ffmpegHelpers] Merge Audio Files Error ${err}`)
+          Logger.error('ffmpeg stdout:', stdout)
+          Logger.error('ffmpeg stderr:', stderr)
+          reject(err)
+        }
+      })
+
+    ffmpeg.run()
+  })
+}
+
+module.exports.mergeAudioFiles = mergeAudioFiles

server/utils/fileUtils.js

@@ -7,24 +7,23 @@ const rra = require('../libs/recursiveReaddirAsync')
 const Logger = require('../Logger')
 const { AudioMimeType } = require('./constants')
 
-
 /**
-* Make sure folder separator is POSIX for Windows file paths. e.g. "C:\Users\Abs" becomes "C:/Users/Abs"
-*
-* @param {String} path - Ugly file path
-* @return {String} Pretty posix file path
-*/
+ * Make sure folder separator is POSIX for Windows file paths. e.g. "C:\Users\Abs" becomes "C:/Users/Abs"
+ *
+ * @param {String} path - Ugly file path
+ * @return {String} Pretty posix file path
+ */
 const filePathToPOSIX = (path) => {
   if (!global.isWin || !path) return path
-  return path.replace(/\\/g, '/')
+  return path.startsWith('\\\\') ? '\\\\' + path.slice(2).replace(/\\/g, '/') : path.replace(/\\/g, '/')
 }
 module.exports.filePathToPOSIX = filePathToPOSIX
 
 /**
  * Check path is a child of or equal to another path
- * 
- * @param {string} parentPath 
- * @param {string} childPath 
+ *
+ * @param {string} parentPath
+ * @param {string} childPath
  * @returns {boolean}
  */
 function isSameOrSubPath(parentPath, childPath) {
@@ -33,8 +32,8 @@ function isSameOrSubPath(parentPath, childPath) {
   if (parentPath === childPath) return true
   const relativePath = Path.relative(parentPath, childPath)
   return (
-    relativePath === '' // Same path (e.g. parentPath = '/a/b/', childPath = '/a/b')
-    || !relativePath.startsWith('..') && !Path.isAbsolute(relativePath) // Sub path
+    relativePath === '' || // Same path (e.g. parentPath = '/a/b/', childPath = '/a/b')
+    (!relativePath.startsWith('..') && !Path.isAbsolute(relativePath)) // Sub path
   )
 }
 module.exports.isSameOrSubPath = isSameOrSubPath
@@ -67,8 +66,8 @@ module.exports.getFileTimestampsWithIno = getFileTimestampsWithIno
 
 /**
  * Get file size
- * 
- * @param {string} path 
+ *
+ * @param {string} path
  * @returns {Promise<number>}
  */
 module.exports.getFileSize = async (path) => {
@@ -77,8 +76,8 @@ module.exports.getFileSize = async (path) => {
 
 /**
  * Get file mtimeMs
- * 
- * @param {string} path 
+ *
+ * @param {string} path
  * @returns {Promise<number>} epoch timestamp
  */
 module.exports.getFileMTimeMs = async (path) => {
@@ -91,8 +90,8 @@ module.exports.getFileMTimeMs = async (path) => {
 }
 
 /**
- * 
- * @param {string} filepath 
+ *
+ * @param {string} filepath
  * @returns {boolean}
  */
 async function checkPathIsFile(filepath) {
@@ -106,16 +105,19 @@ async function checkPathIsFile(filepath) {
 module.exports.checkPathIsFile = checkPathIsFile
 
 function getIno(path) {
-  return fs.stat(path, { bigint: true }).then((data => String(data.ino))).catch((err) => {
-    Logger.error('[Utils] Failed to get ino for path', path, err)
-    return null
-  })
+  return fs
+    .stat(path, { bigint: true })
+    .then((data) => String(data.ino))
+    .catch((err) => {
+      Logger.error('[Utils] Failed to get ino for path', path, err)
+      return null
+    })
 }
 module.exports.getIno = getIno
 
 /**
  * Read contents of file
- * @param {string} path 
+ * @param {string} path
  * @returns {string}
  */
 async function readTextFile(path) {
@@ -129,23 +131,10 @@ async function readTextFile(path) {
 }
 module.exports.readTextFile = readTextFile
 
-function bytesPretty(bytes, decimals = 0) {
-  if (bytes === 0) {
-    return '0 Bytes'
-  }
-  const k = 1000
-  var dm = decimals < 0 ? 0 : decimals
-  const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
-  const i = Math.floor(Math.log(bytes) / Math.log(k))
-  if (i > 2 && dm === 0) dm = 1
-  return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]
-}
-module.exports.bytesPretty = bytesPretty
-
 /**
  * Get array of files inside dir
- * @param {string} path 
- * @param {string} [relPathToReplace] 
+ * @param {string} path
+ * @param {string} [relPathToReplace]
  * @returns {{name:string, path:string, dirpath:string, reldirpath:string, fullpath:string, extension:string, deep:number}[]}
  */
 async function recurseFiles(path, relPathToReplace = null) {
@@ -167,7 +156,7 @@ async function recurseFiles(path, relPathToReplace = null) {
     extensions: true,
     deep: true,
     realPath: true,
-    normalizePath: true
+    normalizePath: false
   }
   let list = await rra.list(path, options)
   if (list.error) {
@@ -177,55 +166,60 @@ async function recurseFiles(path, relPathToReplace = null) {
 
   const directoriesToIgnore = []
 
-  list = list.filter((item) => {
-    if (item.error) {
-      Logger.error(`[fileUtils] Recurse files file "${item.fullname}" has error`, item.error)
-      return false
-    }
+  list = list
+    .filter((item) => {
+      if (item.error) {
+        Logger.error(`[fileUtils] Recurse files file "${item.fullname}" has error`, item.error)
+        return false
+      }
 
-    const relpath = item.fullname.replace(relPathToReplace, '')
-    let reldirname = Path.dirname(relpath)
-    if (reldirname === '.') reldirname = ''
-    const dirname = Path.dirname(item.fullname)
+      item.fullname = filePathToPOSIX(item.fullname)
+      item.path = filePathToPOSIX(item.path)
+      const relpath = item.fullname.replace(relPathToReplace, '')
+      let reldirname = Path.dirname(relpath)
+      if (reldirname === '.') reldirname = ''
+      const dirname = Path.dirname(item.fullname)
 
-    // Directory has a file named ".ignore" flag directory and ignore
-    if (item.name === '.ignore' && reldirname && reldirname !== '.' && !directoriesToIgnore.includes(dirname)) {
-      Logger.debug(`[fileUtils] .ignore found - ignoring directory "${reldirname}"`)
-      directoriesToIgnore.push(dirname)
-      return false
-    }
+      // Directory has a file named ".ignore" flag directory and ignore
+      if (item.name === '.ignore' && reldirname && reldirname !== '.' && !directoriesToIgnore.includes(dirname)) {
+        Logger.debug(`[fileUtils] .ignore found - ignoring directory "${reldirname}"`)
+        directoriesToIgnore.push(dirname)
+        return false
+      }
 
-    if (item.extension === '.part') {
-      Logger.debug(`[fileUtils] Ignoring .part file "${relpath}"`)
-      return false
-    }
+      if (item.extension === '.part') {
+        Logger.debug(`[fileUtils] Ignoring .part file "${relpath}"`)
+        return false
+      }
 
-    // Ignore any file if a directory or the filename starts with "."
-    if (relpath.split('/').find(p => p.startsWith('.'))) {
-      Logger.debug(`[fileUtils] Ignoring path has . "${relpath}"`)
-      return false
-    }
+      // Ignore any file if a directory or the filename starts with "."
+      if (relpath.split('/').find((p) => p.startsWith('.'))) {
+        Logger.debug(`[fileUtils] Ignoring path has . "${relpath}"`)
+        return false
+      }
 
-    return true
-  }).filter(item => {
-    // Filter out items in ignore directories
-    if (directoriesToIgnore.some(dir => item.fullname.startsWith(dir))) {
-      Logger.debug(`[fileUtils] Ignoring path in dir with .ignore "${item.fullname}"`)
-      return false
-    }
-    return true
-  }).map((item) => {
-    var isInRoot = (item.path + '/' === relPathToReplace)
-    return {
-      name: item.name,
-      path: item.fullname.replace(relPathToReplace, ''),
-      dirpath: item.path,
-      reldirpath: isInRoot ? '' : item.path.replace(relPathToReplace, ''),
-      fullpath: item.fullname,
-      extension: item.extension,
-      deep: item.deep
-    }
-  })
+      return true
+    })
+    .filter((item) => {
+      // Filter out items in ignore directories
+      if (directoriesToIgnore.some((dir) => item.fullname.startsWith(dir))) {
+        Logger.debug(`[fileUtils] Ignoring path in dir with .ignore "${item.fullname}"`)
+        return false
+      }
+      return true
+    })
+    .map((item) => {
+      var isInRoot = item.path + '/' === relPathToReplace
+      return {
+        name: item.name,
+        path: item.fullname.replace(relPathToReplace, ''),
+        dirpath: item.path,
+        reldirpath: isInRoot ? '' : item.path.replace(relPathToReplace, ''),
+        fullpath: item.fullname,
+        extension: item.extension,
+        deep: item.deep
+      }
+    })
 
   // Sort from least deep to most
   list.sort((a, b) => a.deep - b.deep)
@@ -237,8 +231,8 @@ module.exports.recurseFiles = recurseFiles
 /**
  * Download file from web to local file system
  * Uses SSRF filter to prevent internal URLs
- * 
- * @param {string} url 
+ *
+ * @param {string} url
  * @param {string} filepath path to download the file to
  * @param {Function} [contentTypeFilter] validate content type before writing
  * @returns {Promise}
@@ -250,34 +244,39 @@ module.exports.downloadFile = (url, filepath, contentTypeFilter = null) => {
       url,
       method: 'GET',
       responseType: 'stream',
+      headers: {
+        'User-Agent': 'audiobookshelf (+https://audiobookshelf.org)'
+      },
       timeout: 30000,
-      httpAgent: ssrfFilter(url),
-      httpsAgent: ssrfFilter(url)
-    }).then((response) => {
-      // Validate content type
-      if (contentTypeFilter && !contentTypeFilter?.(response.headers?.['content-type'])) {
-        return reject(new Error(`Invalid content type "${response.headers?.['content-type'] || ''}"`))
-      }
-
-      // Write to filepath
-      const writer = fs.createWriteStream(filepath)
-      response.data.pipe(writer)
-
-      writer.on('finish', resolve)
-      writer.on('error', reject)
-    }).catch((err) => {
-      Logger.error(`[fileUtils] Failed to download file "${filepath}"`, err)
-      reject(err)
+      httpAgent: global.DisableSsrfRequestFilter ? null : ssrfFilter(url),
+      httpsAgent: global.DisableSsrfRequestFilter ? null : ssrfFilter(url)
     })
+      .then((response) => {
+        // Validate content type
+        if (contentTypeFilter && !contentTypeFilter?.(response.headers?.['content-type'])) {
+          return reject(new Error(`Invalid content type "${response.headers?.['content-type'] || ''}"`))
+        }
+
+        // Write to filepath
+        const writer = fs.createWriteStream(filepath)
+        response.data.pipe(writer)
+
+        writer.on('finish', resolve)
+        writer.on('error', reject)
+      })
+      .catch((err) => {
+        Logger.error(`[fileUtils] Failed to download file "${filepath}"`, err)
+        reject(err)
+      })
   })
 }
 
 /**
  * Download image file from web to local file system
  * Response header must have content-type of image/ (excluding svg)
- * 
- * @param {string} url 
- * @param {string} filepath 
+ *
+ * @param {string} url
+ * @param {string} filepath
  * @returns {Promise}
  */
 module.exports.downloadImageFile = (url, filepath) => {
@@ -350,14 +349,17 @@ module.exports.getAudioMimeTypeFromExtname = (extname) => {
 
 module.exports.removeFile = (path) => {
   if (!path) return false
-  return fs.remove(path).then(() => true).catch((error) => {
-    Logger.error(`[fileUtils] Failed remove file "${path}"`, error)
-    return false
-  })
+  return fs
+    .remove(path)
+    .then(() => true)
+    .catch((error) => {
+      Logger.error(`[fileUtils] Failed remove file "${path}"`, error)
+      return false
+    })
 }
 
 module.exports.encodeUriPath = (path) => {
-  const uri = new URL('/', "file://")
+  const uri = new URL('/', 'file://')
   // we assign the path here to assure that URL control characters like # are
   // actually interpreted as part of the URL path
   uri.pathname = path
@@ -367,8 +369,8 @@ module.exports.encodeUriPath = (path) => {
 /**
  * Check if directory is writable.
  * This method is necessary because fs.access(directory, fs.constants.W_OK) does not work on Windows
- * 
- * @param {string} directory 
+ *
+ * @param {string} directory
  * @returns {Promise<boolean>}
  */
 module.exports.isWritable = async (directory) => {
@@ -385,7 +387,7 @@ module.exports.isWritable = async (directory) => {
 
 /**
  * Get Windows drives as array e.g. ["C:/", "F:/"]
- * 
+ *
  * @returns {Promise<string[]>}
  */
 module.exports.getWindowsDrives = async () => {
@@ -398,7 +400,11 @@ module.exports.getWindowsDrives = async () => {
         reject(error)
         return
       }
-      let drives = stdout?.split(/\r?\n/).map(line => line.trim()).filter(line => line).slice(1)
+      let drives = stdout
+        ?.split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => line)
+        .slice(1)
       const validDrives = []
       for (const drive of drives) {
         let drivepath = drive + '/'
@@ -415,33 +421,78 @@ module.exports.getWindowsDrives = async () => {
 
 /**
  * Get array of directory paths in a directory
- * 
- * @param {string} dirPath 
+ *
+ * @param {string} dirPath
  * @param {number} level
  * @returns {Promise<{ path:string, dirname:string, level:number }[]>}
  */
 module.exports.getDirectoriesInPath = async (dirPath, level) => {
   try {
     const paths = await fs.readdir(dirPath)
-    let dirs = await Promise.all(paths.map(async dirname => {
-      const fullPath = Path.join(dirPath, dirname)
+    let dirs = await Promise.all(
+      paths.map(async (dirname) => {
+        const fullPath = Path.join(dirPath, dirname)
 
-      const lstat = await fs.lstat(fullPath).catch((error) => {
-        Logger.debug(`Failed to lstat "${fullPath}"`, error)
-        return null
+        const lstat = await fs.lstat(fullPath).catch((error) => {
+          Logger.debug(`Failed to lstat "${fullPath}"`, error)
+          return null
+        })
+        if (!lstat?.isDirectory()) return null
+
+        return {
+          path: this.filePathToPOSIX(fullPath),
+          dirname,
+          level
+        }
       })
-      if (!lstat?.isDirectory()) return null
-
-      return {
-        path: this.filePathToPOSIX(fullPath),
-        dirname,
-        level
-      }
-    }))
-    dirs = dirs.filter(d => d)
+    )
+    dirs = dirs.filter((d) => d)
     return dirs
   } catch (error) {
     Logger.error('Failed to readdir', dirPath, error)
     return []
   }
-}
+}
+
+/**
+ * Copies a file from the source path to an existing destination path, preserving the destination's permissions.
+ *
+ * @param {string} srcPath - The path of the source file.
+ * @param {string} destPath - The path of the existing destination file.
+ * @returns {Promise<void>} A promise that resolves when the file has been successfully copied.
+ * @throws {Error} If there is an error reading the source file or writing the destination file.
+ */
+async function copyToExisting(srcPath, destPath) {
+  return new Promise((resolve, reject) => {
+    // Create a readable stream from the source file
+    const readStream = fs.createReadStream(srcPath)
+
+    // Create a writable stream to the destination file
+    const writeStream = fs.createWriteStream(destPath, { flags: 'w' })
+
+    // Pipe the read stream to the write stream
+    readStream.pipe(writeStream)
+
+    // Handle the end of the stream
+    writeStream.on('finish', () => {
+      Logger.debug(`[copyToExisting] Successfully copied file from ${srcPath} to ${destPath}`)
+      resolve()
+    })
+
+    // Handle errors
+    readStream.on('error', (error) => {
+      Logger.error(`[copyToExisting] Error reading from source file ${srcPath}: ${error.message}`)
+      readStream.close()
+      writeStream.close()
+      reject(error)
+    })
+
+    writeStream.on('error', (error) => {
+      Logger.error(`[copyToExisting] Error writing to destination file ${destPath}: ${error.message}`)
+      readStream.close()
+      writeStream.close()
+      reject(error)
+    })
+  })
+}
+module.exports.copyToExisting = copyToExisting

server/utils/generators/abmetadataGenerator.js

@@ -1,4 +1,5 @@
 const Logger = require('../../Logger')
+const parseSeriesString = require('../parsers/parseSeriesString')
 
 function parseJsonMetadataText(text) {
   try {
@@ -19,39 +20,25 @@ function parseJsonMetadataText(text) {
     delete abmetadataData.metadata
 
     if (abmetadataData.series?.length) {
-      abmetadataData.series = [...new Set(abmetadataData.series.map(t => t?.trim()).filter(t => t))]
-      abmetadataData.series = abmetadataData.series.map(series => {
-        let sequence = null
-        let name = series
-        // Series sequence match any characters after " #" other than whitespace and another #
-        //  e.g. "Name #1a" is valid. "Name #1#a" or "Name #1 a" is not valid.
-        const matchResults = series.match(/ #([^#\s]+)$/) // Pull out sequence #
-        if (matchResults && matchResults.length && matchResults.length > 1) {
-          sequence = matchResults[1] // Group 1
-          name = series.replace(matchResults[0], '')
-        }
-        return {
-          name,
-          sequence
-        }
-      })
+      abmetadataData.series = [...new Set(abmetadataData.series.map((t) => t?.trim()).filter((t) => t))]
+      abmetadataData.series = abmetadataData.series.map((series) => parseSeriesString.parse(series))
     }
     // clean tags & remove dupes
     if (abmetadataData.tags?.length) {
-      abmetadataData.tags = [...new Set(abmetadataData.tags.map(t => t?.trim()).filter(t => t))]
+      abmetadataData.tags = [...new Set(abmetadataData.tags.map((t) => t?.trim()).filter((t) => t))]
     }
     if (abmetadataData.chapters?.length) {
       abmetadataData.chapters = cleanChaptersArray(abmetadataData.chapters, abmetadataData.title)
     }
     // clean remove dupes
     if (abmetadataData.authors?.length) {
-      abmetadataData.authors = [...new Set(abmetadataData.authors.map(t => t?.trim()).filter(t => t))]
+      abmetadataData.authors = [...new Set(abmetadataData.authors.map((t) => t?.trim()).filter((t) => t))]
     }
     if (abmetadataData.narrators?.length) {
-      abmetadataData.narrators = [...new Set(abmetadataData.narrators.map(t => t?.trim()).filter(t => t))]
+      abmetadataData.narrators = [...new Set(abmetadataData.narrators.map((t) => t?.trim()).filter((t) => t))]
     }
     if (abmetadataData.genres?.length) {
-      abmetadataData.genres = [...new Set(abmetadataData.genres.map(t => t?.trim()).filter(t => t))]
+      abmetadataData.genres = [...new Set(abmetadataData.genres.map((t) => t?.trim()).filter((t) => t))]
     }
     return abmetadataData
   } catch (error) {

server/utils/globals.js

@@ -1,8 +1,7 @@
 const globals = {
   SupportedImageTypes: ['png', 'jpg', 'jpeg', 'webp'],
-  SupportedAudioTypes: ['m4b', 'mp3', 'm4a', 'flac', 'opus', 'ogg', 'oga', 'mp4', 'aac', 'wma', 'aiff', 'wav', 'webm', 'webma', 'mka', 'awb', 'caf'],
+  SupportedAudioTypes: ['m4b', 'mp3', 'm4a', 'flac', 'opus', 'ogg', 'oga', 'mp4', 'aac', 'wma', 'aiff', 'wav', 'webm', 'webma', 'mka', 'awb', 'caf', 'mpg', 'mpeg'],
   SupportedEbookTypes: ['epub', 'pdf', 'mobi', 'azw3', 'cbr', 'cbz'],
-  SupportedVideoTypes: ['mp4'],
   TextFileTypes: ['txt', 'nfo'],
   MetadataFileTypes: ['opf', 'abs', 'xml', 'json']
 }

server/utils/index.js

@@ -1,7 +1,7 @@
 const Path = require('path')
 const uuid = require('uuid')
 const Logger = require('../Logger')
-const { parseString } = require("xml2js")
+const { parseString } = require('xml2js')
 const areEquivalent = require('./areEquivalent')
 
 const levenshteinDistance = (str1, str2, caseSensitive = false) => {
@@ -11,8 +11,9 @@ const levenshteinDistance = (str1, str2, caseSensitive = false) => {
     str1 = str1.toLowerCase()
     str2 = str2.toLowerCase()
   }
-  const track = Array(str2.length + 1).fill(null).map(() =>
-    Array(str1.length + 1).fill(null))
+  const track = Array(str2.length + 1)
+    .fill(null)
+    .map(() => Array(str1.length + 1).fill(null))
   for (let i = 0; i <= str1.length; i += 1) {
     track[0][i] = i
   }
@@ -25,7 +26,7 @@ const levenshteinDistance = (str1, str2, caseSensitive = false) => {
       track[j][i] = Math.min(
         track[j][i - 1] + 1, // deletion
         track[j - 1][i] + 1, // insertion
-        track[j - 1][i - 1] + indicator, // substitution
+        track[j - 1][i - 1] + indicator // substitution
       )
     }
   }
@@ -65,6 +66,11 @@ module.exports.getId = (prepend = '') => {
   return _id
 }
 
+/**
+ *
+ * @param {number} seconds
+ * @returns {string}
+ */
 function elapsedPretty(seconds) {
   if (seconds > 0 && seconds < 1) {
     return `${Math.floor(seconds * 1000)} ms`
@@ -72,16 +78,27 @@ function elapsedPretty(seconds) {
   if (seconds < 60) {
     return `${Math.floor(seconds)} sec`
   }
-  var minutes = Math.floor(seconds / 60)
+  let minutes = Math.floor(seconds / 60)
   if (minutes < 70) {
     return `${minutes} min`
   }
-  var hours = Math.floor(minutes / 60)
+  let hours = Math.floor(minutes / 60)
   minutes -= hours * 60
-  if (!minutes) {
-    return `${hours} hr`
+
+  let days = Math.floor(hours / 24)
+  hours -= days * 24
+
+  const timeParts = []
+  if (days) {
+    timeParts.push(`${days} d`)
   }
-  return `${hours} hr ${minutes} min`
+  if (hours || (days && minutes)) {
+    timeParts.push(`${hours} hr`)
+  }
+  if (minutes) {
+    timeParts.push(`${minutes} min`)
+  }
+  return timeParts.join(' ')
 }
 module.exports.elapsedPretty = elapsedPretty
 
@@ -138,7 +155,10 @@ module.exports.toNumber = (val, fallback = 0) => {
 module.exports.cleanStringForSearch = (str) => {
   if (!str) return ''
   // Remove ' . ` " ,
-  return str.toLowerCase().replace(/[\'\.\`\",]/g, '').trim()
+  return str
+    .toLowerCase()
+    .replace(/[\'\.\`\",]/g, '')
+    .trim()
 }
 
 const getTitleParts = (title) => {
@@ -156,7 +176,7 @@ const getTitleParts = (title) => {
 /**
  * Remove sortingPrefixes from title
  * @example "The Good Book" => "Good Book"
- * @param {string} title 
+ * @param {string} title
  * @returns {string}
  */
 module.exports.getTitleIgnorePrefix = (title) => {
@@ -164,9 +184,9 @@ module.exports.getTitleIgnorePrefix = (title) => {
 }
 
 /**
- * Put sorting prefix at the end of title 
+ * Put sorting prefix at the end of title
  * @example "The Good Book" => "Good Book, The"
- * @param {string} title 
+ * @param {string} title
  * @returns {string}
  */
 module.exports.getTitlePrefixAtEnd = (title) => {
@@ -174,34 +194,11 @@ module.exports.getTitlePrefixAtEnd = (title) => {
   return prefix ? `${sort}, ${prefix}` : title
 }
 
-/**
- * to lower case for only ascii characters
- * used to handle sqlite that doesnt support unicode lower
- * @see https://github.com/advplyr/audiobookshelf/issues/2187
- * 
- * @param {string} str 
- * @returns {string}
- */
-module.exports.asciiOnlyToLowerCase = (str) => {
-  if (!str) return ''
-
-  let temp = ''
-  for (let chars of str) {
-    let value = chars.charCodeAt()
-    if (value >= 65 && value <= 90) {
-      temp += String.fromCharCode(value + 32)
-    } else {
-      temp += chars
-    }
-  }
-  return temp
-}
-
 /**
  * Escape string used in RegExp
  * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping
- * 
- * @param {string} str 
+ *
+ * @param {string} str
  * @returns {string}
  */
 module.exports.escapeRegExp = (str) => {
@@ -211,8 +208,8 @@ module.exports.escapeRegExp = (str) => {
 
 /**
  * Validate url string with URL class
- * 
- * @param {string} rawUrl 
+ *
+ * @param {string} rawUrl
  * @returns {string} null if invalid
  */
 module.exports.validateUrl = (rawUrl) => {
@@ -227,11 +224,22 @@ module.exports.validateUrl = (rawUrl) => {
 
 /**
  * Check if a string is a valid UUID
- * 
- * @param {string} str 
+ *
+ * @param {string} str
  * @returns {boolean}
  */
 module.exports.isUUID = (str) => {
   if (!str || typeof str !== 'string') return false
   return uuid.validate(str)
-}
+}
+
+/**
+ * Check if a string is a valid ASIN
+ *
+ * @param {string} str
+ * @returns {boolean}
+ */
+module.exports.isValidASIN = (str) => {
+  if (!str || typeof str !== 'string') return false
+  return /^[A-Z0-9]{10}$/.test(str)
+}

server/utils/libraryHelpers.js

@@ -11,7 +11,7 @@ module.exports = {
     const seriesToFilterOut = {}
     books.forEach((libraryItem) => {
       // get all book series for item that is not already filtered out
-      const bookSeries = (libraryItem.media.metadata.series || []).filter(se => !seriesToFilterOut[se.id])
+      const bookSeries = (libraryItem.media.metadata.series || []).filter((se) => !seriesToFilterOut[se.id])
       if (!bookSeries.length) return
 
       bookSeries.forEach((bookSeriesObj) => {
@@ -43,11 +43,11 @@ module.exports = {
 
     // Library setting to hide series with only 1 book
     if (hideSingleBookSeries) {
-      seriesItems = seriesItems.filter(se => se.books.length > 1)
+      seriesItems = seriesItems.filter((se) => se.books.length > 1)
     }
 
     return seriesItems.map((series) => {
-      series.books = naturalSort(series.books).asc(li => li.sequence)
+      series.books = naturalSort(series.books).asc((li) => li.sequence)
       return series
     })
   },
@@ -55,9 +55,7 @@ module.exports = {
   collapseBookSeries(libraryItems, filterSeries, hideSingleBookSeries) {
     // Get series from the library items. If this list is being collapsed after filtering for a series,
     // don't collapse that series, only books that are in other series.
-    const seriesObjects = this
-      .getSeriesFromBooks(libraryItems, filterSeries, hideSingleBookSeries)
-      .filter(s => s.id != filterSeries)
+    const seriesObjects = this.getSeriesFromBooks(libraryItems, filterSeries, hideSingleBookSeries).filter((s) => s.id != filterSeries)
 
     const filteredLibraryItems = []
 
@@ -65,22 +63,29 @@ module.exports = {
       if (li.mediaType != 'book') return
 
       // Handle when this is the first book in a series
-      seriesObjects.filter(s => s.books[0].id == li.id).forEach(series => {
-        // Clone the library item as we need to attach data to it, but don't
-        // want to change the global copy of the library item
-        filteredLibraryItems.push(Object.assign(
-          Object.create(Object.getPrototypeOf(li)),
-          li, { collapsedSeries: series }))
-      })
+      seriesObjects
+        .filter((s) => s.books[0].id == li.id)
+        .forEach((series) => {
+          // Clone the library item as we need to attach data to it, but don't
+          // want to change the global copy of the library item
+          filteredLibraryItems.push(Object.assign(Object.create(Object.getPrototypeOf(li)), li, { collapsedSeries: series }))
+        })
 
       // Only included books not contained in series
-      if (!seriesObjects.some(s => s.books.some(b => b.id == li.id)))
-        filteredLibraryItems.push(li)
+      if (!seriesObjects.some((s) => s.books.some((b) => b.id == li.id))) filteredLibraryItems.push(li)
     })
 
     return filteredLibraryItems
   },
 
+  /**
+   *
+   * @param {*} payload
+   * @param {string} seriesId
+   * @param {import('../models/User')} user
+   * @param {import('../models/Library')} library
+   * @returns {Object[]}
+   */
   async handleCollapseSubseries(payload, seriesId, user, library) {
     const seriesWithBooks = await Database.seriesModel.findByPk(seriesId, {
       include: {
@@ -112,17 +117,18 @@ module.exports = {
       return []
     }
 
-
     const books = seriesWithBooks.books
     payload.total = books.length
 
-    let libraryItems = books.map((book) => {
-      const libraryItem = book.libraryItem
-      libraryItem.media = book
-      return Database.libraryItemModel.getOldLibraryItem(libraryItem)
-    }).filter(li => {
-      return user.checkCanAccessLibraryItem(li)
-    })
+    let libraryItems = books
+      .map((book) => {
+        const libraryItem = book.libraryItem
+        libraryItem.media = book
+        return Database.libraryItemModel.getOldLibraryItem(libraryItem)
+      })
+      .filter((li) => {
+        return user.checkCanAccessLibraryItem(li)
+      })
 
     const collapsedItems = this.collapseBookSeries(libraryItems, seriesId, library.settings.hideSingleBookSeries)
     if (!(collapsedItems.length == 1 && collapsedItems[0].collapsedSeries)) {
@@ -139,7 +145,8 @@ module.exports = {
         {
           [direction]: (li) => li.media.metadata.getSeries(seriesId).sequence
         },
-        { // If no series sequence then fallback to sorting by title (or collapsed series name for sub-series)
+        {
+          // If no series sequence then fallback to sorting by title (or collapsed series name for sub-series)
           [direction]: (li) => {
             if (sortingIgnorePrefix) {
               return li.collapsedSeries?.nameIgnorePrefix || li.media.metadata.titleIgnorePrefix
@@ -150,7 +157,7 @@ module.exports = {
         }
       ]
     } else {
-      // If series are collapsed and not sorting by title or sequence, 
+      // If series are collapsed and not sorting by title or sequence,
       // sort all collapsed series to the end in alphabetical order
       if (payload.sortBy !== 'media.metadata.title') {
         sortArray.push({
@@ -185,47 +192,48 @@ module.exports = {
       libraryItems = libraryItems.slice(startIndex, startIndex + payload.limit)
     }
 
-    return Promise.all(libraryItems.map(async li => {
-      const filteredSeries = li.media.metadata.getSeries(seriesId)
-      const json = li.toJSONMinified()
-      json.media.metadata.series = {
-        id: filteredSeries.id,
-        name: filteredSeries.name,
-        sequence: filteredSeries.sequence
-      }
-
-      if (li.collapsedSeries) {
-        json.collapsedSeries = {
-          id: li.collapsedSeries.id,
-          name: li.collapsedSeries.name,
-          nameIgnorePrefix: li.collapsedSeries.nameIgnorePrefix,
-          libraryItemIds: li.collapsedSeries.books.map(b => b.id),
-          numBooks: li.collapsedSeries.books.length
+    return Promise.all(
+      libraryItems.map(async (li) => {
+        const filteredSeries = li.media.metadata.getSeries(seriesId)
+        const json = li.toJSONMinified()
+        json.media.metadata.series = {
+          id: filteredSeries.id,
+          name: filteredSeries.name,
+          sequence: filteredSeries.sequence
         }
 
-        // If collapsing by series and filtering by a series, generate the list of sequences the collapsed
-        // series represents in the filtered series
-        json.collapsedSeries.seriesSequenceList =
-          naturalSort(li.collapsedSeries.books.filter(b => b.filterSeriesSequence).map(b => b.filterSeriesSequence)).asc()
+        if (li.collapsedSeries) {
+          json.collapsedSeries = {
+            id: li.collapsedSeries.id,
+            name: li.collapsedSeries.name,
+            nameIgnorePrefix: li.collapsedSeries.nameIgnorePrefix,
+            libraryItemIds: li.collapsedSeries.books.map((b) => b.id),
+            numBooks: li.collapsedSeries.books.length
+          }
+
+          // If collapsing by series and filtering by a series, generate the list of sequences the collapsed
+          // series represents in the filtered series
+          json.collapsedSeries.seriesSequenceList = naturalSort(li.collapsedSeries.books.filter((b) => b.filterSeriesSequence).map((b) => b.filterSeriesSequence))
+            .asc()
             .reduce((ranges, currentSequence) => {
               let lastRange = ranges.at(-1)
               let isNumber = /^(\d+|\d+\.\d*|\d*\.\d+)$/.test(currentSequence)
               if (isNumber) currentSequence = parseFloat(currentSequence)
 
-              if (lastRange && isNumber && lastRange.isNumber && ((lastRange.end + 1) == currentSequence)) {
+              if (lastRange && isNumber && lastRange.isNumber && lastRange.end + 1 == currentSequence) {
                 lastRange.end = currentSequence
-              }
-              else {
+              } else {
                 ranges.push({ start: currentSequence, end: currentSequence, isNumber: isNumber })
               }
 
               return ranges
             }, [])
-            .map(r => r.start == r.end ? r.start : `${r.start}-${r.end}`)
+            .map((r) => (r.start == r.end ? r.start : `${r.start}-${r.end}`))
             .join(', ')
-      }
+        }
 
-      return json
-    }))
+        return json
+      })
+    )
   }
 }

server/utils/longTimeout.js

@@ -0,0 +1,36 @@
+/**
+ * Handle timeouts greater than 32-bit signed integer
+ */
+class LongTimeout {
+  constructor() {
+    this.timeout = 0
+    this.timer = null
+  }
+
+  clear() {
+    clearTimeout(this.timer)
+  }
+
+  /**
+   *
+   * @param {Function} fn
+   * @param {number} timeout
+   */
+  set(fn, timeout) {
+    const maxValue = 2147483647
+
+    const handleTimeout = () => {
+      if (this.timeout > 0) {
+        let delay = Math.min(this.timeout, maxValue)
+        this.timeout = this.timeout - delay
+        this.timer = setTimeout(handleTimeout, delay)
+        return
+      }
+      fn()
+    }
+
+    this.timeout = timeout
+    handleTimeout()
+  }
+}
+module.exports = LongTimeout

server/utils/migrations/dbMigration.js

@@ -1,6 +1,6 @@
 const { DataTypes, QueryInterface } = require('sequelize')
 const Path = require('path')
-const uuidv4 = require("uuid").v4
+const uuidv4 = require('uuid').v4
 const Logger = require('../../Logger')
 const fs = require('../../libs/fsExtra')
 const oldDbFiles = require('./oldDbFiles')
@@ -36,25 +36,14 @@ function getDeviceInfoString(deviceInfo, UserId) {
   if (!deviceInfo) return null
   if (deviceInfo.deviceId) return deviceInfo.deviceId
 
-  const keys = [
-    UserId,
-    deviceInfo.browserName || null,
-    deviceInfo.browserVersion || null,
-    deviceInfo.osName || null,
-    deviceInfo.osVersion || null,
-    deviceInfo.clientVersion || null,
-    deviceInfo.manufacturer || null,
-    deviceInfo.model || null,
-    deviceInfo.sdkVersion || null,
-    deviceInfo.ipAddress || null
-  ].map(k => k || '')
+  const keys = [UserId, deviceInfo.browserName || null, deviceInfo.browserVersion || null, deviceInfo.osName || null, deviceInfo.osVersion || null, deviceInfo.clientVersion || null, deviceInfo.manufacturer || null, deviceInfo.model || null, deviceInfo.sdkVersion || null, deviceInfo.ipAddress || null].map((k) => k || '')
   return 'temp-' + Buffer.from(keys.join('-'), 'utf-8').toString('base64')
 }
 
 /**
  * Migrate oldLibraryItem.media to Book model
  * Migrate BookSeries and BookAuthor
- * @param {objects.LibraryItem} oldLibraryItem 
+ * @param {objects.LibraryItem} oldLibraryItem
  * @param {object} LibraryItem models.LibraryItem object
  * @returns {object} { book: object, bookSeries: [], bookAuthor: [] }
  */
@@ -67,7 +56,7 @@ function migrateBook(oldLibraryItem, LibraryItem) {
     bookAuthor: []
   }
 
-  const tracks = (oldBook.audioFiles || []).filter(af => !af.exclude && !af.invalid)
+  const tracks = (oldBook.audioFiles || []).filter((af) => !af.exclude && !af.invalid)
   let duration = 0
   for (const track of tracks) {
     if (track.duration !== null && !isNaN(track.duration)) {
@@ -156,7 +145,7 @@ function migrateBook(oldLibraryItem, LibraryItem) {
 /**
  * Migrate oldLibraryItem.media to Podcast model
  * Migrate PodcastEpisode
- * @param {objects.LibraryItem} oldLibraryItem 
+ * @param {objects.LibraryItem} oldLibraryItem
  * @param {object} LibraryItem models.LibraryItem object
  * @returns {object} { podcast: object, podcastEpisode: [] }
  */
@@ -241,7 +230,7 @@ function migratePodcast(oldLibraryItem, LibraryItem) {
 
 /**
  * Migrate libraryItems to LibraryItem, Book, Podcast models
- * @param {Array<objects.LibraryItem>} oldLibraryItems 
+ * @param {Array<objects.LibraryItem>} oldLibraryItems
  * @returns {object} { libraryItem: [], book: [], podcast: [], podcastEpisode: [], bookSeries: [], bookAuthor: [] }
  */
 function migrateLibraryItems(oldLibraryItems) {
@@ -300,7 +289,7 @@ function migrateLibraryItems(oldLibraryItems) {
       updatedAt: oldLibraryItem.updatedAt,
       libraryId,
       libraryFolderId,
-      libraryFiles: oldLibraryItem.libraryFiles.map(lf => {
+      libraryFiles: oldLibraryItem.libraryFiles.map((lf) => {
         if (lf.isSupplementary === undefined) lf.isSupplementary = null
         return lf
       })
@@ -308,7 +297,7 @@ function migrateLibraryItems(oldLibraryItems) {
     oldDbIdMap.libraryItems[oldLibraryItem.id] = LibraryItem.id
     _newRecords.libraryItem.push(LibraryItem)
 
-    // 
+    //
     // Migrate Book/Podcast
     //
     if (oldLibraryItem.mediaType === 'book') {
@@ -331,7 +320,7 @@ function migrateLibraryItems(oldLibraryItems) {
 
 /**
  * Migrate Library and LibraryFolder
- * @param {Array<objects.Library>} oldLibraries 
+ * @param {Array<objects.Library>} oldLibraries
  * @returns {object} { library: [], libraryFolder: [] }
  */
 function migrateLibraries(oldLibraries) {
@@ -345,7 +334,7 @@ function migrateLibraries(oldLibraries) {
       continue
     }
 
-    // 
+    //
     // Migrate Library
     //
     const Library = {
@@ -363,7 +352,7 @@ function migrateLibraries(oldLibraries) {
     oldDbIdMap.libraries[oldLibrary.id] = Library.id
     _newRecords.library.push(Library)
 
-    // 
+    //
     // Migrate LibraryFolders
     //
     for (const oldFolder of oldLibrary.folders) {
@@ -384,21 +373,27 @@ function migrateLibraries(oldLibraries) {
 /**
  * Migrate Author
  * Previously Authors were shared between libraries, this will ensure every author has one library
- * @param {Array<objects.entities.Author>} oldAuthors 
- * @param {Array<objects.LibraryItem>} oldLibraryItems 
+ * @param {Array<objects.entities.Author>} oldAuthors
+ * @param {Array<objects.LibraryItem>} oldLibraryItems
  * @returns {Array<object>} Array of Author model objs
  */
 function migrateAuthors(oldAuthors, oldLibraryItems) {
   const _newRecords = []
   for (const oldAuthor of oldAuthors) {
     // Get an array of NEW library ids that have this author
-    const librariesWithThisAuthor = [...new Set(oldLibraryItems.map(li => {
-      if (!li.media.metadata.authors?.some(au => au.id === oldAuthor.id)) return null
-      if (!oldDbIdMap.libraries[li.libraryId]) {
-        Logger.warn(`[dbMigration] Authors library id ${li.libraryId} was not migrated`)
-      }
-      return oldDbIdMap.libraries[li.libraryId]
-    }).filter(lid => lid))]
+    const librariesWithThisAuthor = [
+      ...new Set(
+        oldLibraryItems
+          .map((li) => {
+            if (!li.media.metadata.authors?.some((au) => au.id === oldAuthor.id)) return null
+            if (!oldDbIdMap.libraries[li.libraryId]) {
+              Logger.warn(`[dbMigration] Authors library id ${li.libraryId} was not migrated`)
+            }
+            return oldDbIdMap.libraries[li.libraryId]
+          })
+          .filter((lid) => lid)
+      )
+    ]
 
     if (!librariesWithThisAuthor.length) {
       Logger.error(`[dbMigration] Author ${oldAuthor.name} was not found in any libraries`)
@@ -428,8 +423,8 @@ function migrateAuthors(oldAuthors, oldLibraryItems) {
 /**
  * Migrate Series
  * Previously Series were shared between libraries, this will ensure every series has one library
- * @param {Array<objects.entities.Series>} oldSerieses 
- * @param {Array<objects.LibraryItem>} oldLibraryItems 
+ * @param {Array<objects.entities.Series>} oldSerieses
+ * @param {Array<objects.LibraryItem>} oldLibraryItems
  * @returns {Array<object>} Array of Series model objs
  */
 function migrateSeries(oldSerieses, oldLibraryItems) {
@@ -438,10 +433,16 @@ function migrateSeries(oldSerieses, oldLibraryItems) {
   // Series will be separate between libraries
   for (const oldSeries of oldSerieses) {
     // Get an array of NEW library ids that have this series
-    const librariesWithThisSeries = [...new Set(oldLibraryItems.map(li => {
-      if (!li.media.metadata.series?.some(se => se.id === oldSeries.id)) return null
-      return oldDbIdMap.libraries[li.libraryId]
-    }).filter(lid => lid))]
+    const librariesWithThisSeries = [
+      ...new Set(
+        oldLibraryItems
+          .map((li) => {
+            if (!li.media.metadata.series?.some((se) => se.id === oldSeries.id)) return null
+            return oldDbIdMap.libraries[li.libraryId]
+          })
+          .filter((lid) => lid)
+      )
+    ]
 
     if (!librariesWithThisSeries.length) {
       Logger.error(`[dbMigration] Series ${oldSeries.name} was not found in any libraries`)
@@ -467,7 +468,7 @@ function migrateSeries(oldSerieses, oldLibraryItems) {
 
 /**
  * Migrate users to User and MediaProgress models
- * @param {Array<objects.User>} oldUsers 
+ * @param {Array<objects.User>} oldUsers
  * @returns {object} { user: [], mediaProgress: [] }
  */
 function migrateUsers(oldUsers) {
@@ -476,29 +477,33 @@ function migrateUsers(oldUsers) {
     mediaProgress: []
   }
   for (const oldUser of oldUsers) {
-    // 
+    //
     // Migrate User
     //
     // Convert old library ids to new ids
-    const librariesAccessible = (oldUser.librariesAccessible || []).map((lid) => oldDbIdMap.libraries[lid]).filter(li => li)
+    const librariesAccessible = (oldUser.librariesAccessible || []).map((lid) => oldDbIdMap.libraries[lid]).filter((li) => li)
 
     // Convert old library item ids to new ids
-    const bookmarks = (oldUser.bookmarks || []).map(bm => {
-      bm.libraryItemId = oldDbIdMap.libraryItems[bm.libraryItemId]
-      return bm
-    }).filter(bm => bm.libraryItemId)
+    const bookmarks = (oldUser.bookmarks || [])
+      .map((bm) => {
+        bm.libraryItemId = oldDbIdMap.libraryItems[bm.libraryItemId]
+        return bm
+      })
+      .filter((bm) => bm.libraryItemId)
 
     // Convert old series ids to new
-    const seriesHideFromContinueListening = (oldUser.seriesHideFromContinueListening || []).map(oldSeriesId => {
-      // Series were split to be per library
-      // This will use the first series it finds
-      for (const libraryId in oldDbIdMap.series) {
-        if (oldDbIdMap.series[libraryId][oldSeriesId]) {
-          return oldDbIdMap.series[libraryId][oldSeriesId]
+    const seriesHideFromContinueListening = (oldUser.seriesHideFromContinueListening || [])
+      .map((oldSeriesId) => {
+        // Series were split to be per library
+        // This will use the first series it finds
+        for (const libraryId in oldDbIdMap.series) {
+          if (oldDbIdMap.series[libraryId][oldSeriesId]) {
+            return oldDbIdMap.series[libraryId][oldSeriesId]
+          }
         }
-      }
-      return null
-    }).filter(se => se)
+        return null
+      })
+      .filter((se) => se)
 
     const User = {
       id: uuidv4(),
@@ -523,7 +528,7 @@ function migrateUsers(oldUsers) {
     oldDbIdMap.users[oldUser.id] = User.id
     _newRecords.user.push(User)
 
-    // 
+    //
     // Migrate MediaProgress
     //
     for (const oldMediaProgress of oldUser.mediaProgress) {
@@ -568,7 +573,7 @@ function migrateUsers(oldUsers) {
 
 /**
  * Migrate playbackSessions to PlaybackSession and Device models
- * @param {Array<objects.PlaybackSession>} oldSessions 
+ * @param {Array<objects.PlaybackSession>} oldSessions
  * @returns {object} { playbackSession: [], device: [] }
  */
 function migrateSessions(oldSessions) {
@@ -692,7 +697,7 @@ function migrateSessions(oldSessions) {
 
 /**
  * Migrate collections to Collection & CollectionBook
- * @param {Array<objects.Collection>} oldCollections 
+ * @param {Array<objects.Collection>} oldCollections
  * @returns {object} { collection: [], collectionBook: [] }
  */
 function migrateCollections(oldCollections) {
@@ -707,7 +712,7 @@ function migrateCollections(oldCollections) {
       continue
     }
 
-    const BookIds = oldCollection.books.map(lid => oldDbIdMap.books[lid]).filter(bid => bid)
+    const BookIds = oldCollection.books.map((lid) => oldDbIdMap.books[lid]).filter((bid) => bid)
     if (!BookIds.length) {
       Logger.warn(`[dbMigration] migrateCollections: Collection "${oldCollection.name}" has no books`)
       continue
@@ -741,7 +746,7 @@ function migrateCollections(oldCollections) {
 
 /**
  * Migrate playlists to Playlist and PlaylistMediaItem
- * @param {Array<objects.Playlist>} oldPlaylists 
+ * @param {Array<objects.Playlist>} oldPlaylists
  * @returns {object} { playlist: [], playlistMediaItem: [] }
  */
 function migratePlaylists(oldPlaylists) {
@@ -808,7 +813,7 @@ function migratePlaylists(oldPlaylists) {
 
 /**
  * Migrate feeds to Feed and FeedEpisode models
- * @param {Array<objects.Feed>} oldFeeds 
+ * @param {Array<objects.Feed>} oldFeeds
  * @returns {object} { feed: [], feedEpisode: [] }
  */
 function migrateFeeds(oldFeeds) {
@@ -909,14 +914,14 @@ function migrateFeeds(oldFeeds) {
 
 /**
  * Migrate ServerSettings, NotificationSettings and EmailSettings to Setting model
- * @param {Array<objects.settings.*>} oldSettings 
+ * @param {Array<objects.settings.*>} oldSettings
  * @returns {Array<object>} Array of Setting model objs
  */
 function migrateSettings(oldSettings) {
   const _newRecords = []
-  const serverSettings = oldSettings.find(s => s.id === 'server-settings')
-  const notificationSettings = oldSettings.find(s => s.id === 'notification-settings')
-  const emailSettings = oldSettings.find(s => s.id === 'email-settings')
+  const serverSettings = oldSettings.find((s) => s.id === 'server-settings')
+  const notificationSettings = oldSettings.find((s) => s.id === 'notification-settings')
+  const emailSettings = oldSettings.find((s) => s.id === 'email-settings')
 
   if (serverSettings) {
     _newRecords.push({
@@ -948,7 +953,7 @@ function migrateSettings(oldSettings) {
 
 /**
  * Load old libraries and bulkCreate new Library and LibraryFolder rows
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  */
 async function handleMigrateLibraries(DatabaseModels) {
   const oldLibraries = await oldDbFiles.loadOldData('libraries')
@@ -961,7 +966,7 @@ async function handleMigrateLibraries(DatabaseModels) {
 
 /**
  * Load old EmailSettings, NotificationSettings and ServerSettings and bulkCreate new Setting rows
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  */
 async function handleMigrateSettings(DatabaseModels) {
   const oldSettings = await oldDbFiles.loadOldData('settings')
@@ -972,7 +977,7 @@ async function handleMigrateSettings(DatabaseModels) {
 
 /**
  * Load old authors and bulkCreate new Author rows
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  * @param {Array<objects.LibraryItem>} oldLibraryItems
  */
 async function handleMigrateAuthors(DatabaseModels, oldLibraryItems) {
@@ -984,7 +989,7 @@ async function handleMigrateAuthors(DatabaseModels, oldLibraryItems) {
 
 /**
  * Load old series and bulkCreate new Series rows
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  * @param {Array<objects.LibraryItem>} oldLibraryItems
  */
 async function handleMigrateSeries(DatabaseModels, oldLibraryItems) {
@@ -996,7 +1001,7 @@ async function handleMigrateSeries(DatabaseModels, oldLibraryItems) {
 
 /**
  * bulkCreate new LibraryItem, Book and Podcast rows
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  * @param {Array<objects.LibraryItem>} oldLibraryItems
  */
 async function handleMigrateLibraryItems(DatabaseModels, oldLibraryItems) {
@@ -1010,7 +1015,7 @@ async function handleMigrateLibraryItems(DatabaseModels, oldLibraryItems) {
 /**
  * Migrate authors, series then library items in chunks
  * Authors and series require old library items loaded first
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  */
 async function handleMigrateAuthorsSeriesAndLibraryItems(DatabaseModels) {
   const oldLibraryItems = await oldDbFiles.loadOldData('libraryItems')
@@ -1028,7 +1033,7 @@ async function handleMigrateAuthorsSeriesAndLibraryItems(DatabaseModels) {
 
 /**
  * Load old users and bulkCreate new User rows
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  */
 async function handleMigrateUsers(DatabaseModels) {
   const oldUsers = await oldDbFiles.loadOldData('users')
@@ -1041,7 +1046,7 @@ async function handleMigrateUsers(DatabaseModels) {
 
 /**
  * Load old sessions and bulkCreate new PlaybackSession & Device rows
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  */
 async function handleMigrateSessions(DatabaseModels) {
   const oldSessions = await oldDbFiles.loadOldData('sessions')
@@ -1057,12 +1062,11 @@ async function handleMigrateSessions(DatabaseModels) {
       await DatabaseModels[model].bulkCreate(newSessionRecords[model])
     }
   }
-
 }
 
 /**
  * Load old collections and bulkCreate new Collection, CollectionBook models
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  */
 async function handleMigrateCollections(DatabaseModels) {
   const oldCollections = await oldDbFiles.loadOldData('collections')
@@ -1075,7 +1079,7 @@ async function handleMigrateCollections(DatabaseModels) {
 
 /**
  * Load old playlists and bulkCreate new Playlist, PlaylistMediaItem models
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  */
 async function handleMigratePlaylists(DatabaseModels) {
   const oldPlaylists = await oldDbFiles.loadOldData('playlists')
@@ -1088,7 +1092,7 @@ async function handleMigratePlaylists(DatabaseModels) {
 
 /**
  * Load old feeds and bulkCreate new Feed, FeedEpisode models
- * @param {Map<string,Model>} DatabaseModels 
+ * @param {Map<string,Model>} DatabaseModels
  */
 async function handleMigrateFeeds(DatabaseModels) {
   const oldFeeds = await oldDbFiles.loadOldData('feeds')
@@ -1154,21 +1158,36 @@ module.exports.checkShouldMigrate = async () => {
 
 /**
  * Migration from 2.3.0 to 2.3.1 - create extraData columns in LibraryItem and PodcastEpisode
- * @param {QueryInterface} queryInterface 
+ * @param {QueryInterface} queryInterface
  */
 async function migrationPatchNewColumns(queryInterface) {
   try {
-    return queryInterface.sequelize.transaction(t => {
+    return queryInterface.sequelize.transaction((t) => {
       return Promise.all([
-        queryInterface.addColumn('libraryItems', 'extraData', {
-          type: DataTypes.JSON
-        }, { transaction: t }),
-        queryInterface.addColumn('podcastEpisodes', 'extraData', {
-          type: DataTypes.JSON
-        }, { transaction: t }),
-        queryInterface.addColumn('libraries', 'extraData', {
-          type: DataTypes.JSON
-        }, { transaction: t })
+        queryInterface.addColumn(
+          'libraryItems',
+          'extraData',
+          {
+            type: DataTypes.JSON
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          'podcastEpisodes',
+          'extraData',
+          {
+            type: DataTypes.JSON
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          'libraries',
+          'extraData',
+          {
+            type: DataTypes.JSON
+          },
+          { transaction: t }
+        )
       ])
     })
   } catch (error) {
@@ -1179,7 +1198,7 @@ async function migrationPatchNewColumns(queryInterface) {
 
 /**
  * Migration from 2.3.0 to 2.3.1 - old library item ids
- * @param {/src/Database} ctx 
+ * @param {/src/Database} ctx
  */
 async function handleOldLibraryItems(ctx) {
   const oldLibraryItems = await oldDbFiles.loadOldData('libraryItems')
@@ -1190,7 +1209,7 @@ async function handleOldLibraryItems(ctx) {
 
   for (const libraryItem of libraryItems) {
     // Find matching old library item by ino
-    const matchingOldLibraryItem = oldLibraryItems.find(oli => oli.ino === libraryItem.ino)
+    const matchingOldLibraryItem = oldLibraryItems.find((oli) => oli.ino === libraryItem.ino)
     if (matchingOldLibraryItem) {
       oldDbIdMap.libraryItems[matchingOldLibraryItem.id] = libraryItem.id
 
@@ -1204,7 +1223,7 @@ async function handleOldLibraryItems(ctx) {
       if (libraryItem.media.episodes?.length && matchingOldLibraryItem.media.episodes?.length) {
         for (const podcastEpisode of libraryItem.media.episodes) {
           // Find matching old episode by audio file ino
-          const matchingOldPodcastEpisode = matchingOldLibraryItem.media.episodes.find(oep => oep.audioFile?.ino && oep.audioFile.ino === podcastEpisode.audioFile?.ino)
+          const matchingOldPodcastEpisode = matchingOldLibraryItem.media.episodes.find((oep) => oep.audioFile?.ino && oep.audioFile.ino === podcastEpisode.audioFile?.ino)
           if (matchingOldPodcastEpisode) {
             oldDbIdMap.podcastEpisodes[matchingOldPodcastEpisode.id] = podcastEpisode.id
 
@@ -1237,27 +1256,31 @@ async function handleOldLibraryItems(ctx) {
 
 /**
  * Migration from 2.3.0 to 2.3.1 - updating oldLibraryId
- * @param {/src/Database} ctx 
+ * @param {/src/Database} ctx
  */
 async function handleOldLibraries(ctx) {
   const oldLibraries = await oldDbFiles.loadOldData('libraries')
-  const libraries = await ctx.models.library.getAllOldLibraries()
+  const libraries = await ctx.models.library.getAllWithFolders()
 
   let librariesUpdated = 0
   for (const library of libraries) {
     // Find matching old library using exact match on folder paths, exact match on library name
-    const matchingOldLibrary = oldLibraries.find(ol => {
+    const matchingOldLibrary = oldLibraries.find((ol) => {
       if (ol.name !== library.name) {
         return false
       }
-      const folderPaths = ol.folders?.map(f => f.fullPath) || []
-      return folderPaths.join(',') === library.folderPaths.join(',')
+      const folderPaths = ol.folders?.map((f) => f.fullPath) || []
+      return folderPaths.join(',') === library.libraryFolders.map((f) => f.path).join(',')
     })
 
     if (matchingOldLibrary) {
-      library.oldLibraryId = matchingOldLibrary.id
+      const newExtraData = library.extraData || {}
+      newExtraData.oldLibraryId = matchingOldLibrary.id
+      library.extraData = newExtraData
+      library.changed('extraData', true)
+
       oldDbIdMap.libraries[library.oldLibraryId] = library.id
-      await ctx.models.library.updateFromOld(library)
+      await library.save()
       librariesUpdated++
     }
   }
@@ -1266,46 +1289,67 @@ async function handleOldLibraries(ctx) {
 
 /**
  * Migration from 2.3.0 to 2.3.1 - fixing librariesAccessible and bookmarks
- * @param {/src/Database} ctx 
+ * @param {import('../../Database')} ctx
  */
 async function handleOldUsers(ctx) {
-  const users = await ctx.models.user.getOldUsers()
+  const usersNew = await ctx.userModel.findAll({
+    include: ctx.models.mediaProgress
+  })
 
   let usersUpdated = 0
-  for (const user of users) {
+  for (const user of usersNew) {
     let hasUpdates = false
     if (user.bookmarks?.length) {
-      user.bookmarks = user.bookmarks.map(bm => {
-        // Only update if this is not the old id format
-        if (!bm.libraryItemId.startsWith('li_')) return bm
+      user.bookmarks = user.bookmarks
+        .map((bm) => {
+          // Only update if this is not the old id format
+          if (!bm.libraryItemId.startsWith('li_')) return bm
 
-        bm.libraryItemId = oldDbIdMap.libraryItems[bm.libraryItemId]
-        hasUpdates = true
-        return bm
-      }).filter(bm => bm.libraryItemId)
+          bm.libraryItemId = oldDbIdMap.libraryItems[bm.libraryItemId]
+          hasUpdates = true
+          return bm
+        })
+        .filter((bm) => bm.libraryItemId)
+      if (hasUpdates) {
+        user.changed('bookmarks', true)
+      }
     }
 
+    const librariesAccessible = user.permissions?.librariesAccessible || []
+
     // Convert old library ids to new library ids
-    if (user.librariesAccessible?.length) {
-      user.librariesAccessible = user.librariesAccessible.map(lid => {
-        if (!lid.startsWith('lib_') && lid !== 'main') return lid // Already not an old library id so dont change
-        hasUpdates = true
-        return oldDbIdMap.libraries[lid]
-      }).filter(lid => lid)
+    if (librariesAccessible.length) {
+      user.permissions.librariesAccessible = librariesAccessible
+        .map((lid) => {
+          if (!lid.startsWith('lib_') && lid !== 'main') return lid // Already not an old library id so dont change
+          hasUpdates = true
+          return oldDbIdMap.libraries[lid]
+        })
+        .filter((lid) => lid)
+      if (hasUpdates) {
+        user.changed('permissions', true)
+      }
     }
 
-    if (user.seriesHideFromContinueListening?.length) {
-      user.seriesHideFromContinueListening = user.seriesHideFromContinueListening.map((seriesId) => {
-        if (seriesId.startsWith('se_')) {
-          hasUpdates = true
-          return null // Filter out old series ids
-        }
-        return seriesId
-      }).filter(se => se)
+    const seriesHideFromContinueListening = user.extraData?.seriesHideFromContinueListening || []
+
+    if (seriesHideFromContinueListening.length) {
+      user.extraData.seriesHideFromContinueListening = seriesHideFromContinueListening
+        .map((seriesId) => {
+          if (seriesId.startsWith('se_')) {
+            hasUpdates = true
+            return null // Filter out old series ids
+          }
+          return seriesId
+        })
+        .filter((se) => se)
+      if (hasUpdates) {
+        user.changed('extraData', true)
+      }
     }
 
     if (hasUpdates) {
-      await ctx.models.user.updateFromOld(user)
+      await user.save()
       usersUpdated++
     }
   }
@@ -1314,7 +1358,7 @@ async function handleOldUsers(ctx) {
 
 /**
  * Migration from 2.3.0 to 2.3.1
- * @param {/src/Database} ctx 
+ * @param {/src/Database} ctx
  */
 module.exports.migrationPatch = async (ctx) => {
   const queryInterface = ctx.sequelize.getQueryInterface()
@@ -1330,7 +1374,7 @@ module.exports.migrationPatch = async (ctx) => {
   }
 
   const oldDbPath = Path.join(global.ConfigPath, 'oldDb.zip')
-  if (!await fs.pathExists(oldDbPath)) {
+  if (!(await fs.pathExists(oldDbPath))) {
     Logger.info(`[dbMigration] Migration patch 2.3.0+ unnecessary - no oldDb.zip found`)
     return
   }
@@ -1339,7 +1383,7 @@ module.exports.migrationPatch = async (ctx) => {
   Logger.info(`[dbMigration] Applying migration patch from 2.3.0+`)
 
   // Extract from oldDb.zip
-  if (!await oldDbFiles.checkExtractItemsUsersAndLibraries()) {
+  if (!(await oldDbFiles.checkExtractItemsUsersAndLibraries())) {
     return
   }
 
@@ -1356,8 +1400,8 @@ module.exports.migrationPatch = async (ctx) => {
 /**
  * Migration from 2.3.3 to 2.3.4
  * Populating the size column on libraryItem
- * @param {/src/Database} ctx 
- * @param {number} offset 
+ * @param {/src/Database} ctx
+ * @param {number} offset
  */
 async function migrationPatch2LibraryItems(ctx, offset = 0) {
   const libraryItems = await ctx.models.libraryItem.findAll({
@@ -1370,7 +1414,7 @@ async function migrationPatch2LibraryItems(ctx, offset = 0) {
   for (const libraryItem of libraryItems) {
     if (libraryItem.libraryFiles?.length) {
       let size = 0
-      libraryItem.libraryFiles.forEach(lf => {
+      libraryItem.libraryFiles.forEach((lf) => {
         if (!isNaN(lf.metadata?.size)) {
           size += Number(lf.metadata.size)
         }
@@ -1398,8 +1442,8 @@ async function migrationPatch2LibraryItems(ctx, offset = 0) {
 /**
  * Migration from 2.3.3 to 2.3.4
  * Populating the duration & titleIgnorePrefix column on book
- * @param {/src/Database} ctx 
- * @param {number} offset 
+ * @param {/src/Database} ctx
+ * @param {number} offset
  */
 async function migrationPatch2Books(ctx, offset = 0) {
   const books = await ctx.models.book.findAll({
@@ -1413,7 +1457,7 @@ async function migrationPatch2Books(ctx, offset = 0) {
     let duration = 0
 
     if (book.audioFiles?.length) {
-      const tracks = book.audioFiles.filter(af => !af.exclude && !af.invalid)
+      const tracks = book.audioFiles.filter((af) => !af.exclude && !af.invalid)
       for (const track of tracks) {
         if (track.duration !== null && !isNaN(track.duration)) {
           duration += track.duration
@@ -1444,8 +1488,8 @@ async function migrationPatch2Books(ctx, offset = 0) {
 /**
  * Migration from 2.3.3 to 2.3.4
  * Populating the titleIgnorePrefix column on podcast
- * @param {/src/Database} ctx 
- * @param {number} offset 
+ * @param {/src/Database} ctx
+ * @param {number} offset
  */
 async function migrationPatch2Podcasts(ctx, offset = 0) {
   const podcasts = await ctx.models.podcast.findAll({
@@ -1478,8 +1522,8 @@ async function migrationPatch2Podcasts(ctx, offset = 0) {
 /**
  * Migration from 2.3.3 to 2.3.4
  * Populating the nameIgnorePrefix column on series
- * @param {/src/Database} ctx 
- * @param {number} offset 
+ * @param {/src/Database} ctx
+ * @param {number} offset
  */
 async function migrationPatch2Series(ctx, offset = 0) {
   const allSeries = await ctx.models.series.findAll({
@@ -1512,8 +1556,8 @@ async function migrationPatch2Series(ctx, offset = 0) {
 /**
  * Migration from 2.3.3 to 2.3.4
  * Populating the lastFirst column on author
- * @param {/src/Database} ctx 
- * @param {number} offset 
+ * @param {/src/Database} ctx
+ * @param {number} offset
  */
 async function migrationPatch2Authors(ctx, offset = 0) {
   const authors = await ctx.models.author.findAll({
@@ -1548,8 +1592,8 @@ async function migrationPatch2Authors(ctx, offset = 0) {
 /**
  * Migration from 2.3.3 to 2.3.4
  * Populating the createdAt column on bookAuthor
- * @param {/src/Database} ctx 
- * @param {number} offset 
+ * @param {/src/Database} ctx
+ * @param {number} offset
  */
 async function migrationPatch2BookAuthors(ctx, offset = 0) {
   const bookAuthors = await ctx.models.bookAuthor.findAll({
@@ -1583,8 +1627,8 @@ async function migrationPatch2BookAuthors(ctx, offset = 0) {
 /**
  * Migration from 2.3.3 to 2.3.4
  * Populating the createdAt column on bookSeries
- * @param {/src/Database} ctx 
- * @param {number} offset 
+ * @param {/src/Database} ctx
+ * @param {number} offset
  */
 async function migrationPatch2BookSeries(ctx, offset = 0) {
   const allBookSeries = await ctx.models.bookSeries.findAll({
@@ -1618,7 +1662,7 @@ async function migrationPatch2BookSeries(ctx, offset = 0) {
 /**
  * Migration from 2.3.3 to 2.3.4
  * Adding coverPath column to Feed model
- * @param {/src/Database} ctx 
+ * @param {/src/Database} ctx
  */
 module.exports.migrationPatch2 = async (ctx) => {
   const queryInterface = ctx.sequelize.getQueryInterface()
@@ -1633,44 +1677,95 @@ module.exports.migrationPatch2 = async (ctx) => {
   Logger.info(`[dbMigration] Applying migration patch from 2.3.3+`)
 
   try {
-    await queryInterface.sequelize.transaction(t => {
+    await queryInterface.sequelize.transaction((t) => {
       const queries = []
       if (!bookAuthorsTableDescription?.createdAt) {
-        queries.push(...[
-          queryInterface.addColumn('bookAuthors', 'createdAt', {
-            type: DataTypes.DATE
-          }, { transaction: t }),
-          queryInterface.addColumn('bookSeries', 'createdAt', {
-            type: DataTypes.DATE
-          }, { transaction: t }),
-        ])
+        queries.push(
+          ...[
+            queryInterface.addColumn(
+              'bookAuthors',
+              'createdAt',
+              {
+                type: DataTypes.DATE
+              },
+              { transaction: t }
+            ),
+            queryInterface.addColumn(
+              'bookSeries',
+              'createdAt',
+              {
+                type: DataTypes.DATE
+              },
+              { transaction: t }
+            )
+          ]
+        )
       }
       if (!authorsTableDescription?.lastFirst) {
-        queries.push(...[
-          queryInterface.addColumn('authors', 'lastFirst', {
-            type: DataTypes.STRING
-          }, { transaction: t }),
-          queryInterface.addColumn('libraryItems', 'size', {
-            type: DataTypes.BIGINT
-          }, { transaction: t }),
-          queryInterface.addColumn('books', 'duration', {
-            type: DataTypes.FLOAT
-          }, { transaction: t }),
-          queryInterface.addColumn('books', 'titleIgnorePrefix', {
-            type: DataTypes.STRING
-          }, { transaction: t }),
-          queryInterface.addColumn('podcasts', 'titleIgnorePrefix', {
-            type: DataTypes.STRING
-          }, { transaction: t }),
-          queryInterface.addColumn('series', 'nameIgnorePrefix', {
-            type: DataTypes.STRING
-          }, { transaction: t }),
-        ])
+        queries.push(
+          ...[
+            queryInterface.addColumn(
+              'authors',
+              'lastFirst',
+              {
+                type: DataTypes.STRING
+              },
+              { transaction: t }
+            ),
+            queryInterface.addColumn(
+              'libraryItems',
+              'size',
+              {
+                type: DataTypes.BIGINT
+              },
+              { transaction: t }
+            ),
+            queryInterface.addColumn(
+              'books',
+              'duration',
+              {
+                type: DataTypes.FLOAT
+              },
+              { transaction: t }
+            ),
+            queryInterface.addColumn(
+              'books',
+              'titleIgnorePrefix',
+              {
+                type: DataTypes.STRING
+              },
+              { transaction: t }
+            ),
+            queryInterface.addColumn(
+              'podcasts',
+              'titleIgnorePrefix',
+              {
+                type: DataTypes.STRING
+              },
+              { transaction: t }
+            ),
+            queryInterface.addColumn(
+              'series',
+              'nameIgnorePrefix',
+              {
+                type: DataTypes.STRING
+              },
+              { transaction: t }
+            )
+          ]
+        )
       }
       if (!feedTableDescription?.coverPath) {
-        queries.push(queryInterface.addColumn('feeds', 'coverPath', {
-          type: DataTypes.STRING
-        }, { transaction: t }))
+        queries.push(
+          queryInterface.addColumn(
+            'feeds',
+            'coverPath',
+            {
+              type: DataTypes.STRING
+            },
+            { transaction: t }
+          )
+        )
       }
       return Promise.all(queries)
     })

server/utils/notifications.js

@@ -7,6 +7,7 @@ module.exports.notificationData = {
       requiresLibrary: true,
       libraryMediaType: 'podcast',
       description: 'Triggered when a podcast episode is auto-downloaded',
+      descriptionKey: 'NotificationOnEpisodeDownloadedDescription',
       variables: ['libraryItemId', 'libraryId', 'podcastTitle', 'podcastAuthor', 'podcastDescription', 'podcastGenres', 'episodeTitle', 'episodeSubtitle', 'episodeDescription', 'libraryName', 'episodeId', 'mediaTags'],
       defaults: {
         title: 'New {{podcastTitle}} Episode!',
@@ -27,10 +28,43 @@ module.exports.notificationData = {
         episodeDescription: 'Some description of the podcast episode.'
       }
     },
+    {
+      name: 'onBackupCompleted',
+      requiresLibrary: false,
+      description: 'Triggered when a backup is completed',
+      descriptionKey: 'NotificationOnBackupCompletedDescription',
+      variables: ['completionTime', 'backupPath', 'backupSize', 'backupCount', 'removedOldest'],
+      defaults: {
+        title: 'Backup Completed',
+        body: 'Backup has been completed successfully.\n\nPath: {{backupPath}}\nSize: {{backupSize}} bytes\nCount: {{backupCount}}\nRemoved Oldest: {{removedOldest}}'
+      },
+      testData: {
+        completionTime: '12:00 AM',
+        backupPath: 'path/to/backup',
+        backupSize: '1.23 MB',
+        backupCount: '1',
+        removedOldest: 'false'
+      }
+    },
+    {
+      name: 'onBackupFailed',
+      requiresLibrary: false,
+      description: 'Triggered when a backup fails',
+      descriptionKey: 'NotificationOnBackupFailedDescription',
+      variables: ['errorMsg'],
+      defaults: {
+        title: 'Backup Failed',
+        body: 'Backup failed, check ABS logs for more information.\nError message: {{errorMsg}}'
+      },
+      testData: {
+        errorMsg: 'Example error message'
+      }
+    },
     {
       name: 'onTest',
       requiresLibrary: false,
       description: 'Event for testing the notification system',
+      descriptionKey: 'NotificationOnTestDescription',
       variables: ['version'],
       defaults: {
         title: 'Test Notification on Abs {{version}}',

server/utils/parsers/parseComicMetadata.js

@@ -1,109 +1,91 @@
 const Path = require('path')
-const globals = require('../globals')
-const fs = require('../../libs/fsExtra')
 const Logger = require('../../Logger')
-const Archive = require('../../libs/libarchive/archive')
-const { xmlToJSON } = require('../index')
 const parseComicInfoMetadata = require('./parseComicInfoMetadata')
-
-/**
- * 
- * @param {string} filepath 
- * @returns {Promise<Buffer>}
- */
-async function getComicFileBuffer(filepath) {
-  if (!await fs.pathExists(filepath)) {
-    Logger.error(`Comic path does not exist "${filepath}"`)
-    return null
-  }
-  try {
-    return fs.readFile(filepath)
-  } catch (error) {
-    Logger.error(`Failed to read comic at "${filepath}"`, error)
-    return null
-  }
-}
+const globals = require('../globals')
+const { xmlToJSON } = require('../index')
+const { createComicBookExtractor } = require('../comicBookExtractors.js')
 
 /**
  * Extract cover image from comic return true if success
- * 
- * @param {string} comicPath 
- * @param {string} comicImageFilepath 
- * @param {string} outputCoverPath 
+ *
+ * @param {string} comicPath
+ * @param {string} comicImageFilepath
+ * @param {string} outputCoverPath
  * @returns {Promise<boolean>}
  */
 async function extractCoverImage(comicPath, comicImageFilepath, outputCoverPath) {
-  const comicFileBuffer = await getComicFileBuffer(comicPath)
-  if (!comicFileBuffer) return null
-
-  const archive = await Archive.open(comicFileBuffer)
-  const fileEntry = await archive.extractSingleFile(comicImageFilepath)
-
-  if (!fileEntry?.fileData) {
-    Logger.error(`[parseComicMetadata] Invalid file entry data for comicPath "${comicPath}"/${comicImageFilepath}`)
-    return false
-  }
-
+  let archive = null
   try {
-    await fs.writeFile(outputCoverPath, fileEntry.fileData)
-    return true
+    archive = createComicBookExtractor(comicPath)
+    await archive.open()
+    return await archive.extractToFile(comicImageFilepath, outputCoverPath)
   } catch (error) {
-    Logger.error(`[parseComicMetadata] Failed to extract image from comicPath "${comicPath}"`, error)
+    Logger.error(`[parseComicMetadata] Failed to extract image "${comicImageFilepath}" from comicPath "${comicPath}" into "${outputCoverPath}"`, error)
     return false
+  } finally {
+    // Ensure we free the memory
+    archive?.close()
   }
 }
 module.exports.extractCoverImage = extractCoverImage
 
 /**
  * Parse metadata from comic
- * 
- * @param {import('../../models/Book').EBookFileObject} ebookFile 
+ *
+ * @param {import('../../models/Book').EBookFileObject} ebookFile
  * @returns {Promise<import('./parseEbookMetadata').EBookFileScanData>}
  */
 async function parse(ebookFile) {
   const comicPath = ebookFile.metadata.path
-  Logger.debug(`Parsing metadata from comic at "${comicPath}"`)
+  Logger.debug(`[parseComicMetadata] Parsing comic metadata at "${comicPath}"`)
+  let archive = null
+  try {
+    archive = createComicBookExtractor(comicPath)
+    await archive.open()
 
-  const comicFileBuffer = await getComicFileBuffer(comicPath)
-  if (!comicFileBuffer) return null
+    const filePaths = await archive.getFilePaths()
 
-  const archive = await Archive.open(comicFileBuffer)
-
-  const fileObjects = await archive.getFilesArray()
-
-  fileObjects.sort((a, b) => {
-    return a.file.name.localeCompare(b.file.name, undefined, {
-      numeric: true,
-      sensitivity: 'base'
+    // Sort the file paths in a natural order to get the first image
+    filePaths.sort((a, b) => {
+      return a.localeCompare(b, undefined, {
+        numeric: true,
+        sensitivity: 'base'
+      })
     })
-  })
 
-  let metadata = null
-  const comicInfo = fileObjects.find(fo => fo.file.name === 'ComicInfo.xml')
-  if (comicInfo) {
-    const comicInfoEntry = await comicInfo.file.extract()
-    if (comicInfoEntry?.fileData) {
-      const comicInfoStr = new TextDecoder().decode(comicInfoEntry.fileData)
-      const comicInfoJson = await xmlToJSON(comicInfoStr)
-      if (comicInfoJson) {
-        metadata = parseComicInfoMetadata.parse(comicInfoJson)
+    let metadata = null
+    const comicInfoPath = filePaths.find((filePath) => filePath === 'ComicInfo.xml')
+    if (comicInfoPath) {
+      const comicInfoData = await archive.extractToBuffer(comicInfoPath)
+      if (comicInfoData) {
+        const comicInfoStr = new TextDecoder().decode(comicInfoData)
+        const comicInfoJson = await xmlToJSON(comicInfoStr)
+        if (comicInfoJson) {
+          metadata = parseComicInfoMetadata.parse(comicInfoJson)
+        }
       }
     }
-  }
 
-  const payload = {
-    path: comicPath,
-    ebookFormat: ebookFile.ebookFormat,
-    metadata
-  }
+    const payload = {
+      path: comicPath,
+      ebookFormat: ebookFile.ebookFormat,
+      metadata
+    }
 
-  const firstImage = fileObjects.find(fo => globals.SupportedImageTypes.includes(Path.extname(fo.file.name).toLowerCase().slice(1)))
-  if (firstImage?.file?._path) {
-    payload.ebookCoverPath = firstImage.file._path
-  } else {
-    Logger.warn(`Cover image not found in comic at "${comicPath}"`)
-  }
+    const firstImagePath = filePaths.find((filePath) => globals.SupportedImageTypes.includes(Path.extname(filePath).toLowerCase().slice(1)))
+    if (firstImagePath) {
+      payload.ebookCoverPath = firstImagePath
+    } else {
+      Logger.warn(`[parseComicMetadata] Cover image not found in comic at "${comicPath}"`)
+    }
 
-  return payload
+    return payload
+  } catch (error) {
+    Logger.error(`[parseComicMetadata] Failed to parse comic metadata at "${comicPath}"`, error)
+    return null
+  } finally {
+    // Ensure we free the memory
+    archive?.close()
+  }
 }
-module.exports.parse = parse
+module.exports.parse = parse

server/utils/parsers/parseEpubMetadata.js

@@ -4,12 +4,11 @@ const StreamZip = require('../../libs/nodeStreamZip')
 const parseOpfMetadata = require('./parseOpfMetadata')
 const { xmlToJSON } = require('../index')
 
-
 /**
  * Extract file from epub and return string content
- * 
- * @param {string} epubPath 
- * @param {string} filepath 
+ *
+ * @param {string} epubPath
+ * @param {string} filepath
  * @returns {Promise<string>}
  */
 async function extractFileFromEpub(epubPath, filepath) {
@@ -27,9 +26,9 @@ async function extractFileFromEpub(epubPath, filepath) {
 
 /**
  * Extract an XML file from epub and return JSON
- * 
- * @param {string} epubPath 
- * @param {string} xmlFilepath 
+ *
+ * @param {string} epubPath
+ * @param {string} xmlFilepath
  * @returns {Promise<Object>}
  */
 async function extractXmlToJson(epubPath, xmlFilepath) {
@@ -40,19 +39,22 @@ async function extractXmlToJson(epubPath, xmlFilepath) {
 
 /**
  * Extract cover image from epub return true if success
- * 
- * @param {string} epubPath 
- * @param {string} epubImageFilepath 
- * @param {string} outputCoverPath 
+ *
+ * @param {string} epubPath
+ * @param {string} epubImageFilepath
+ * @param {string} outputCoverPath
  * @returns {Promise<boolean>}
  */
 async function extractCoverImage(epubPath, epubImageFilepath, outputCoverPath) {
   const zip = new StreamZip.async({ file: epubPath })
 
-  const success = await zip.extract(epubImageFilepath, outputCoverPath).then(() => true).catch((error) => {
-    Logger.error(`[parseEpubMetadata] Failed to extract image ${epubImageFilepath} from epub at "${epubPath}"`, error)
-    return false
-  })
+  const success = await zip
+    .extract(epubImageFilepath, outputCoverPath)
+    .then(() => true)
+    .catch((error) => {
+      Logger.error(`[parseEpubMetadata] Failed to extract image ${epubImageFilepath} from epub at "${epubPath}"`, error)
+      return false
+    })
 
   await zip.close()
 
@@ -62,8 +64,8 @@ module.exports.extractCoverImage = extractCoverImage
 
 /**
  * Parse metadata from epub
- * 
- * @param {import('../../models/Book').EBookFileObject} ebookFile 
+ *
+ * @param {import('../../models/Book').EBookFileObject} ebookFile
  * @returns {Promise<import('./parseEbookMetadata').EBookFileScanData>}
  */
 async function parse(ebookFile) {
@@ -89,7 +91,7 @@ async function parse(ebookFile) {
   }
 
   // Parse metadata from package document opf file
-  const opfMetadata = parseOpfMetadata.parseOpfMetadataJson(packageJson)
+  const opfMetadata = parseOpfMetadata.parseOpfMetadataJson(structuredClone(packageJson))
   if (!opfMetadata) {
     Logger.error(`Unable to parse metadata in package doc with json`, JSON.stringify(packageJson, null, 2))
     return null
@@ -101,8 +103,23 @@ async function parse(ebookFile) {
     metadata: opfMetadata
   }
 
-  // Attempt to find filepath to cover image
-  const manifestFirstImage = packageJson.package?.manifest?.[0]?.item?.find(item => item.$?.['media-type']?.startsWith('image/'))
+  // Attempt to find filepath to cover image:
+  // Metadata may include <meta name="cover" content="id"/> where content is the id of the cover image in the manifest
+  //  Otherwise the first image in the manifest is used as the cover image
+  let packageMetadata = packageJson.package?.metadata
+  if (Array.isArray(packageMetadata)) {
+    packageMetadata = packageMetadata[0]
+  }
+  const metaCoverId = packageMetadata?.meta?.find?.((meta) => meta.$?.name === 'cover')?.$?.content
+
+  let manifestFirstImage = null
+  if (metaCoverId) {
+    manifestFirstImage = packageJson.package?.manifest?.[0]?.item?.find((item) => item.$?.id === metaCoverId)
+  }
+  if (!manifestFirstImage) {
+    manifestFirstImage = packageJson.package?.manifest?.[0]?.item?.find((item) => item.$?.['media-type']?.startsWith('image/'))
+  }
+
   let coverImagePath = manifestFirstImage?.$?.href
   if (coverImagePath) {
     const packageDirname = Path.dirname(packageDocPath)
@@ -113,4 +130,4 @@ async function parse(ebookFile) {
 
   return payload
 }
-module.exports.parse = parse
+module.exports.parse = parse

server/utils/parsers/parseNameString.js

@@ -42,15 +42,15 @@ module.exports.parse = (nameString) => {
   var splitNames = []
   // Example &LF: Friedman, Milton & Friedman, Rose
   if (nameString.includes('&')) {
-    nameString.split('&').forEach((asa) => splitNames = splitNames.concat(asa.split(',')))
+    nameString.split('&').forEach((asa) => (splitNames = splitNames.concat(asa.split(','))))
   } else if (nameString.includes(' and ')) {
-    nameString.split(' and ').forEach((asa) => splitNames = splitNames.concat(asa.split(',')))
+    nameString.split(' and ').forEach((asa) => (splitNames = splitNames.concat(asa.split(','))))
   } else if (nameString.includes(';')) {
-    nameString.split(';').forEach((asa) => splitNames = splitNames.concat(asa.split(',')))
+    nameString.split(';').forEach((asa) => (splitNames = splitNames.concat(asa.split(','))))
   } else {
     splitNames = nameString.split(',')
   }
-  if (splitNames.length) splitNames = splitNames.map(a => a.trim())
+  if (splitNames.length) splitNames = splitNames.map((a) => a.trim())
 
   var names = []
 
@@ -84,21 +84,12 @@ module.exports.parse = (nameString) => {
   }
 
   // Filter out names that have no first and last
-  names = names.filter(n => n.first_name || n.last_name)
+  names = names.filter((n) => n.first_name || n.last_name)
 
   // Set name strings and remove duplicates
-  const namesArray = [...new Set(names.map(a => a.first_name ? `${a.first_name} ${a.last_name}` : a.last_name))]
+  const namesArray = [...new Set(names.map((a) => (a.first_name ? `${a.first_name} ${a.last_name}` : a.last_name)))]
 
   return {
     names: namesArray // Array of first last
   }
 }
-
-module.exports.checkNamesAreEqual = (name1, name2) => {
-  if (!name1 || !name2) return false
-
-  // e.g. John H. Smith will be equal to John H Smith
-  name1 = String(name1).toLowerCase().trim().replace(/\./g, '')
-  name2 = String(name2).toLowerCase().trim().replace(/\./g, '')
-  return name1 === name2
-}

server/utils/parsers/parseNfoMetadata.js

@@ -81,6 +81,10 @@ function parseNfoMetadata(nfoText) {
         case 'isbn-13':
           metadata.isbn = value
           break
+        case 'language':
+        case 'lang':
+          metadata.language = value
+          break
       }
     }
   })

server/utils/parsers/parseOPML.js

@@ -1,17 +1,21 @@
 const h = require('htmlparser2')
 const Logger = require('../../Logger')
 
+/**
+ *
+ * @param {string} opmlText
+ * @returns {Array<{title: string, feedUrl: string}>
+ */
 function parse(opmlText) {
   var feeds = []
   var parser = new h.Parser({
     onopentag: (name, attribs) => {
-      if (name === "outline" && attribs.type === 'rss') {
+      if (name === 'outline' && attribs.type === 'rss') {
         if (!attribs.xmlurl) {
           Logger.error('[parseOPML] Invalid opml outline tag has no xmlurl attribute')
         } else {
           feeds.push({
-            title: attribs.title || 'No Title',
-            text: attribs.text || '',
+            title: attribs.title || attribs.text || '',
             feedUrl: attribs.xmlurl
           })
         }
@@ -21,4 +25,4 @@ function parse(opmlText) {
   parser.write(opmlText)
   return feeds
 }
-module.exports.parse = parse
+module.exports.parse = parse

server/utils/parsers/parseOpfMetadata.js

@@ -1,23 +1,38 @@
 const { xmlToJSON } = require('../index')
 const htmlSanitizer = require('../htmlSanitizer')
 
+/**
+ * @typedef MetadataCreatorObject
+ * @property {string} value
+ * @property {string} role
+ * @property {string} fileAs
+ *
+ * @example
+ * <dc:creator xmlns:ns0="http://www.idpf.org/2007/opf" ns0:role="aut" ns0:file-as="Steinbeck, John">John Steinbeck</dc:creator>
+ * <dc:creator opf:role="aut" opf:file-as="Orwell, George">George Orwell</dc:creator>
+ *
+ * @param {Object} metadata
+ * @returns {MetadataCreatorObject[]}
+ */
 function parseCreators(metadata) {
-  if (!metadata['dc:creator']) return null
-  const creators = metadata['dc:creator']
-  if (!creators.length) return null
-  return creators.map(c => {
+  if (!metadata['dc:creator']?.length) return null
+  return metadata['dc:creator'].map((c) => {
     if (typeof c !== 'object' || !c['$'] || !c['_']) return false
+    const namespace =
+      Object.keys(c['$'])
+        .find((key) => key.startsWith('xmlns:'))
+        ?.split(':')[1] || 'opf'
     return {
       value: c['_'],
-      role: c['$']['opf:role'] || null,
-      fileAs: c['$']['opf:file-as'] || null
+      role: c['$'][`${namespace}:role`] || null,
+      fileAs: c['$'][`${namespace}:file-as`] || null
     }
   })
 }
 
 function fetchCreators(creators, role) {
   if (!creators?.length) return null
-  return [...new Set(creators.filter(c => c.role === role && c.value).map(c => c.value))]
+  return [...new Set(creators.filter((c) => c.role === role && c.value).map((c) => c.value))]
 }
 
 function fetchTagString(metadata, tag) {
@@ -59,18 +74,34 @@ function fetchPublisher(metadata) {
   return fetchTagString(metadata, 'dc:publisher')
 }
 
+/**
+ * @example
+ * <dc:identifier xmlns:ns4="http://www.idpf.org/2007/opf" ns4:scheme="ISBN">9781440633904</dc:identifier>
+ * <dc:identifier opf:scheme="ISBN">9780141187761</dc:identifier>
+ *
+ * @param {Object} metadata
+ * @param {string} scheme
+ * @returns {string}
+ */
+function fetchIdentifier(metadata, scheme) {
+  if (!metadata['dc:identifier']?.length) return null
+  const identifierObj = metadata['dc:identifier'].find((i) => {
+    if (!i['$']) return false
+    const namespace =
+      Object.keys(i['$'])
+        .find((key) => key.startsWith('xmlns:'))
+        ?.split(':')[1] || 'opf'
+    return i['$'][`${namespace}:scheme`] === scheme
+  })
+  return identifierObj?.['_'] || null
+}
+
 function fetchISBN(metadata) {
-  if (!metadata['dc:identifier'] || !metadata['dc:identifier'].length) return null
-  const identifiers = metadata['dc:identifier']
-  const isbnObj = identifiers.find(i => i['$'] && i['$']['opf:scheme'] === 'ISBN')
-  return isbnObj ? isbnObj['_'] || null : null
+  return fetchIdentifier(metadata, 'ISBN')
 }
 
 function fetchASIN(metadata) {
-  if (!metadata['dc:identifier'] || !metadata['dc:identifier'].length) return null
-  const identifiers = metadata['dc:identifier']
-  const asinObj = identifiers.find(i => i['$'] && i['$']['opf:scheme'] === 'ASIN')
-  return asinObj ? asinObj['_'] || null : null
+  return fetchIdentifier(metadata, 'ASIN')
 }
 
 function fetchTitle(metadata) {
@@ -92,7 +123,7 @@ function fetchDescription(metadata) {
 
 function fetchGenres(metadata) {
   if (!metadata['dc:subject'] || !metadata['dc:subject'].length) return []
-  return [...new Set(metadata['dc:subject'].filter(g => g && typeof g === 'string'))]
+  return [...new Set(metadata['dc:subject'].filter((g) => g && typeof g === 'string'))]
 }
 
 function fetchLanguage(metadata) {
@@ -116,20 +147,24 @@ function fetchSeries(metadataMeta) {
   // If one series was found with no series_index then check if any series_index meta can be found
   //   this is to support when calibre:series_index is not directly underneath calibre:series
   if (result.length === 1 && !result[0].sequence) {
-    const seriesIndexMeta = metadataMeta.find(m => m.$?.name === 'calibre:series_index' && m.$.content?.trim())
+    const seriesIndexMeta = metadataMeta.find((m) => m.$?.name === 'calibre:series_index' && m.$.content?.trim())
     if (seriesIndexMeta) {
       result[0].sequence = seriesIndexMeta.$.content.trim()
     }
   }
-  return result
+
+  // Remove duplicates
+  const dedupedResult = result.filter((se, idx) => result.findIndex((s) => s.name === se.name) === idx)
+
+  return dedupedResult
 }
 
 function fetchNarrators(creators, metadata) {
   const narrators = fetchCreators(creators, 'nrt')
   if (narrators?.length) return narrators
   try {
-    const narratorsJSON = JSON.parse(fetchTagString(metadata.meta, "calibre:user_metadata:#narrators").replace(/&quot;/g, '"'))
-    return narratorsJSON["#value#"]
+    const narratorsJSON = JSON.parse(fetchTagString(metadata.meta, 'calibre:user_metadata:#narrators').replace(/&quot;/g, '"'))
+    return narratorsJSON['#value#']
   } catch {
     return null
   }
@@ -137,7 +172,7 @@ function fetchNarrators(creators, metadata) {
 
 function fetchTags(metadata) {
   if (!metadata['dc:tag'] || !metadata['dc:tag'].length) return []
-  return [...new Set(metadata['dc:tag'].filter(tag => tag && typeof tag === 'string'))]
+  return [...new Set(metadata['dc:tag'].filter((tag) => tag && typeof tag === 'string'))]
 }
 
 function stripPrefix(str) {
@@ -147,7 +182,7 @@ function stripPrefix(str) {
 
 module.exports.parseOpfMetadataJson = (json) => {
   // Handle <package ...> or with prefix <ns0:package ...>
-  const packageKey = Object.keys(json).find(key => stripPrefix(key) === 'package')
+  const packageKey = Object.keys(json).find((key) => stripPrefix(key) === 'package')
   if (!packageKey) return null
   const prefix = packageKey.split(':').shift()
   let metadata = prefix ? json[packageKey][`${prefix}:metadata`] || json[packageKey].metadata : json[packageKey].metadata
@@ -170,8 +205,8 @@ module.exports.parseOpfMetadataJson = (json) => {
   }
 
   const creators = parseCreators(metadata)
-  const authors = (fetchCreators(creators, 'aut') || []).map(au => au?.trim()).filter(au => au)
-  const narrators = (fetchNarrators(creators, metadata) || []).map(nrt => nrt?.trim()).filter(nrt => nrt)
+  const authors = (fetchCreators(creators, 'aut') || []).map((au) => au?.trim()).filter((au) => au)
+  const narrators = (fetchNarrators(creators, metadata) || []).map((nrt) => nrt?.trim()).filter((nrt) => nrt)
   return {
     title: fetchTitle(metadata),
     subtitle: fetchSubtitle(metadata),
@@ -193,4 +228,4 @@ module.exports.parseOpfMetadataXML = async (xml) => {
   const json = await xmlToJSON(xml)
   if (!json) return null
   return this.parseOpfMetadataJson(json)
-}
+}

server/utils/parsers/parseOverdriveMediaMarkers.js

@@ -63,7 +63,7 @@ function cleanOverdriveMediaMarkers(overdriveMediaMarkers) {
 function objectValuesArrayToString(arrayOfObjects) {
   Logger.debug('[parseOverdriveMediaMarkers] Converting Marker object values from arrays to strings')
   arrayOfObjects.forEach((item) => {
-    Object.keys(item).forEach(key => {
+    Object.keys(item).forEach((key) => {
       item[key] = item[key].toString()
     })
   })
@@ -78,7 +78,7 @@ function removeExtraChapters(parsedOverdriveMediaMarkers) {
   const weirdChapterFilterRegex = /([(]\d|[cC]ontinued)/
   var cleaned = []
   parsedOverdriveMediaMarkers.forEach(function (item) {
-    cleaned.push(item.filter(chapter => !weirdChapterFilterRegex.test(chapter.Name)))
+    cleaned.push(item.filter((chapter) => !weirdChapterFilterRegex.test(chapter.Name)))
   })
 
   return cleaned
@@ -110,11 +110,20 @@ function generateParsedChapters(includedAudioFiles, cleanedOverdriveMediaMarkers
 
   // cleanedOverdriveMediaMarkers is an array of array of objects, where the inner array matches to the included audio files tracks
   //     this allows us to leverage the individual track durations when calculating the start times of chapters in tracks after the first (using length)
-  // TODO: can we guarantee the inner array matches the included audio files? 
+  // TODO: can we guarantee the inner array matches the included audio files?
   includedAudioFiles.forEach((track, track_index) => {
     cleanedOverdriveMediaMarkers[track_index].forEach((chapter) => {
-      time = chapter.Time.split(":")
-      time = length + parseFloat(time[0]) * 60 + parseFloat(time[1])
+      let timeParts = chapter.Time.split(':')
+      // add seconds
+      time = length + parseFloat(timeParts.pop())
+      if (timeParts.length) {
+        // add minutes
+        time += parseFloat(timeParts.pop()) * 60
+      }
+      if (timeParts.length) {
+        // add hours
+        time += parseFloat(timeParts.pop()) * 3600
+      }
       var newChapterData = {
         id: index++,
         start: time,
@@ -131,7 +140,7 @@ function generateParsedChapters(includedAudioFiles, cleanedOverdriveMediaMarkers
 }
 
 module.exports.parseOverdriveMediaMarkersAsChapters = (includedAudioFiles) => {
-  const overdriveMediaMarkers = includedAudioFiles.map((af) => af.metaTags.tagOverdriveMediaMarker).filter(af => af) || []
+  const overdriveMediaMarkers = includedAudioFiles.map((af) => af.metaTags.tagOverdriveMediaMarker).filter((af) => af) || []
   if (!overdriveMediaMarkers.length) return null
 
   var cleanedOverdriveMediaMarkers = cleanOverdriveMediaMarkers(overdriveMediaMarkers)
@@ -139,4 +148,4 @@ module.exports.parseOverdriveMediaMarkersAsChapters = (includedAudioFiles) => {
   //         so if not equal then we must exit
   if (cleanedOverdriveMediaMarkers.length !== includedAudioFiles.length) return null
   return generateParsedChapters(includedAudioFiles, cleanedOverdriveMediaMarkers)
-}
+}

server/utils/parsers/parseSeriesString.js

@@ -0,0 +1,27 @@
+/**
+ * Parse a series string into a name and sequence
+ *
+ * @example
+ * Name #1a => { name: 'Name', sequence: '1a' }
+ * Name #1 => { name: 'Name', sequence: '1' }
+ *
+ * @param {string} seriesString
+ * @returns {{name: string, sequence: string}|null}
+ */
+module.exports.parse = (seriesString) => {
+  if (!seriesString || typeof seriesString !== 'string') return null
+
+  let sequence = null
+  let name = seriesString
+  // Series sequence match any characters after " #" other than whitespace and another #
+  //  e.g. "Name #1a" is valid. "Name #1#a" or "Name #1 a" is not valid.
+  const matchResults = seriesString.match(/ #([^#\s]+)$/) // Pull out sequence #
+  if (matchResults && matchResults.length && matchResults.length > 1) {
+    sequence = matchResults[1] // Group 1
+    name = seriesString.replace(matchResults[0], '')
+  }
+  return {
+    name,
+    sequence
+  }
+}

server/utils/podcastUtils.js

@@ -220,8 +220,8 @@ module.exports.parsePodcastRssFeedXml = async (xml, excludeEpisodeMetadata = fal
 /**
  * Get podcast RSS feed as JSON
  * Uses SSRF filter to prevent internal URLs
- * 
- * @param {string} feedUrl 
+ *
+ * @param {string} feedUrl
  * @param {boolean} [excludeEpisodeMetadata=false]
  * @returns {Promise}
  */
@@ -233,38 +233,42 @@ module.exports.getPodcastFeed = (feedUrl, excludeEpisodeMetadata = false) => {
     method: 'GET',
     timeout: 12000,
     responseType: 'arraybuffer',
-    headers: { Accept: 'application/rss+xml, application/xhtml+xml, application/xml, */*;q=0.8' },
-    httpAgent: ssrfFilter(feedUrl),
-    httpsAgent: ssrfFilter(feedUrl)
-  }).then(async (data) => {
-
-    // Adding support for ios-8859-1 encoded RSS feeds.
-    //  See: https://github.com/advplyr/audiobookshelf/issues/1489
-    const contentType = data.headers?.['content-type'] || '' // e.g. text/xml; charset=iso-8859-1
-    if (contentType.toLowerCase().includes('iso-8859-1')) {
-      data.data = data.data.toString('latin1')
-    } else {
-      data.data = data.data.toString()
-    }
-
-    if (!data?.data) {
-      Logger.error(`[podcastUtils] getPodcastFeed: Invalid podcast feed request response (${feedUrl})`)
-      return null
-    }
-    Logger.debug(`[podcastUtils] getPodcastFeed for "${feedUrl}" success - parsing xml`)
-    const payload = await this.parsePodcastRssFeedXml(data.data, excludeEpisodeMetadata)
-    if (!payload) {
-      return null
-    }
-
-    // RSS feed may be a private RSS feed
-    payload.podcast.metadata.feedUrl = feedUrl
-
-    return payload.podcast
-  }).catch((error) => {
-    Logger.error('[podcastUtils] getPodcastFeed Error', error)
-    return null
+    headers: {
+      Accept: 'application/rss+xml, application/xhtml+xml, application/xml, */*;q=0.8',
+      'User-Agent': 'audiobookshelf (+https://audiobookshelf.org; like iTMS)'
+    },
+    httpAgent: global.DisableSsrfRequestFilter ? null : ssrfFilter(feedUrl),
+    httpsAgent: global.DisableSsrfRequestFilter ? null : ssrfFilter(feedUrl)
   })
+    .then(async (data) => {
+      // Adding support for ios-8859-1 encoded RSS feeds.
+      //  See: https://github.com/advplyr/audiobookshelf/issues/1489
+      const contentType = data.headers?.['content-type'] || '' // e.g. text/xml; charset=iso-8859-1
+      if (contentType.toLowerCase().includes('iso-8859-1')) {
+        data.data = data.data.toString('latin1')
+      } else {
+        data.data = data.data.toString()
+      }
+
+      if (!data?.data) {
+        Logger.error(`[podcastUtils] getPodcastFeed: Invalid podcast feed request response (${feedUrl})`)
+        return null
+      }
+      Logger.debug(`[podcastUtils] getPodcastFeed for "${feedUrl}" success - parsing xml`)
+      const payload = await this.parsePodcastRssFeedXml(data.data, excludeEpisodeMetadata)
+      if (!payload) {
+        return null
+      }
+
+      // RSS feed may be a private RSS feed
+      payload.podcast.metadata.feedUrl = feedUrl
+
+      return payload.podcast
+    })
+    .catch((error) => {
+      Logger.error('[podcastUtils] getPodcastFeed Error', error)
+      return null
+    })
 }
 
 // Return array of episodes ordered by closest match (Levenshtein distance of 6 or less)
@@ -283,9 +287,8 @@ module.exports.findMatchingEpisodesInFeed = (feed, searchTitle) => {
   }
 
   const matches = []
-  feed.episodes.forEach(ep => {
+  feed.episodes.forEach((ep) => {
     if (!ep.title) return
-
     const epTitle = ep.title.toLowerCase().trim()
     if (epTitle === searchTitle) {
       matches.push({

server/utils/prober.js

@@ -189,6 +189,7 @@ function parseTags(format, verbose) {
     file_tag_genre: tryGrabTags(format, 'genre', 'tcon', 'tco'),
     file_tag_series: tryGrabTags(format, 'series', 'show', 'mvnm'),
     file_tag_seriespart: tryGrabTags(format, 'series-part', 'episode_id', 'mvin', 'part'),
+    file_tag_grouping: tryGrabTags(format, 'grouping'),
     file_tag_isbn: tryGrabTags(format, 'isbn'), // custom
     file_tag_language: tryGrabTags(format, 'language', 'lang'),
     file_tag_asin: tryGrabTags(format, 'asin', 'audible_asin'), // custom

server/utils/queries/authorFilters.js

@@ -54,18 +54,16 @@ module.exports = {
    * Search authors
    *
    * @param {string} libraryId
-   * @param {string} query
+   * @param {Database.TextQuery} query
    * @param {number} limit
    * @param {number} offset
    * @returns {Promise<Object[]>} oldAuthor with numBooks
    */
   async search(libraryId, query, limit, offset) {
+    const matchAuthor = query.matchExpression('name')
     const authors = await Database.authorModel.findAll({
       where: {
-        name: {
-          [Sequelize.Op.substring]: query
-        },
-        libraryId
+        [Sequelize.Op.and]: [Sequelize.literal(matchAuthor), { libraryId }]
       },
       attributes: {
         include: [[Sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 'numBooks']]
@@ -75,8 +73,7 @@ module.exports = {
     })
     const authorMatches = []
     for (const author of authors) {
-      const oldAuthor = author.getOldAuthor().toJSON()
-      oldAuthor.numBooks = author.dataValues.numBooks
+      const oldAuthor = author.toOldJSONExpanded(author.dataValues.numBooks)
       authorMatches.push(oldAuthor)
     }
     return authorMatches

server/utils/queries/libraryFilters.js

@@ -15,47 +15,50 @@ module.exports = {
 
   /**
    * Get library items using filter and sort
-   * @param {import('../../objects/Library')} library 
-   * @param {import('../../objects/user/User')} user 
-   * @param {object} options 
+   * @param {string} libraryId
+   * @param {import('../../models/User')} user
+   * @param {object} options
    * @returns {object} { libraryItems:LibraryItem[], count:number }
    */
-  async getFilteredLibraryItems(library, user, options) {
+  async getFilteredLibraryItems(libraryId, user, options) {
     const { filterBy, sortBy, sortDesc, limit, offset, collapseseries, include, mediaType } = options
 
     let filterValue = null
     let filterGroup = null
     if (filterBy) {
-      const searchGroups = ['genres', 'tags', 'series', 'authors', 'progress', 'narrators', 'publishers', 'missing', 'languages', 'tracks', 'ebooks']
-      const group = searchGroups.find(_group => filterBy.startsWith(_group + '.'))
+      const searchGroups = ['genres', 'tags', 'series', 'authors', 'progress', 'narrators', 'publishers', 'publishedDecades', 'missing', 'languages', 'tracks', 'ebooks']
+      const group = searchGroups.find((_group) => filterBy.startsWith(_group + '.'))
       filterGroup = group || filterBy
       filterValue = group ? this.decode(filterBy.replace(`${group}.`, '')) : null
     }
 
     if (mediaType === 'book') {
-      return libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, filterGroup, filterValue, sortBy, sortDesc, collapseseries, include, limit, offset)
+      return libraryItemsBookFilters.getFilteredLibraryItems(libraryId, user, filterGroup, filterValue, sortBy, sortDesc, collapseseries, include, limit, offset)
     } else {
-      return libraryItemsPodcastFilters.getFilteredLibraryItems(library.id, user, filterGroup, filterValue, sortBy, sortDesc, include, limit, offset)
+      return libraryItemsPodcastFilters.getFilteredLibraryItems(libraryId, user, filterGroup, filterValue, sortBy, sortDesc, include, limit, offset)
     }
   },
 
   /**
    * Get library items for continue listening & continue reading shelves
-   * @param {import('../../objects/Library')} library 
-   * @param {import('../../objects/user/User')} user 
-   * @param {string[]} include 
-   * @param {number} limit 
+   * @param {import('../../models/Library')} library
+   * @param {import('../../models/User')} user
+   * @param {string[]} include
+   * @param {number} limit
    * @returns {Promise<{ items:import('../../models/LibraryItem')[], count:number }>}
    */
   async getMediaItemsInProgress(library, user, include, limit) {
-    if (library.mediaType === 'book') {
+    if (library.isBook) {
       const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'in-progress', 'progress', true, false, include, limit, 0, true)
       return {
-        items: libraryItems.map(li => {
+        items: libraryItems.map((li) => {
           const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(li).toJSONMinified()
           if (li.rssFeed) {
             oldLibraryItem.rssFeed = Database.feedModel.getOldFeed(li.rssFeed).toJSONMinified()
           }
+          if (li.mediaItemShare) {
+            oldLibraryItem.mediaItemShare = li.mediaItemShare
+          }
           return oldLibraryItem
         }),
         count
@@ -64,7 +67,7 @@ module.exports = {
       const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'in-progress', 'progress', true, limit, 0, true)
       return {
         count,
-        items: libraryItems.map(li => {
+        items: libraryItems.map((li) => {
           const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(li).toJSONMinified()
           oldLibraryItem.recentEpisode = li.recentEpisode
           return oldLibraryItem
@@ -75,17 +78,17 @@ module.exports = {
 
   /**
    * Get library items for most recently added shelf
-   * @param {import('../../objects/Library')} library 
-   * @param {oldUser} user 
-   * @param {string[]} include 
-   * @param {number} limit 
+   * @param {import('../../models/Library')} library
+   * @param {import('../../models/User')} user
+   * @param {string[]} include
+   * @param {number} limit
    * @returns {object} { libraryItems:LibraryItem[], count:number }
    */
   async getLibraryItemsMostRecentlyAdded(library, user, include, limit) {
-    if (library.mediaType === 'book') {
+    if (library.isBook) {
       const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, false, include, limit, 0)
       return {
-        libraryItems: libraryItems.map(li => {
+        libraryItems: libraryItems.map((li) => {
           const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(li).toJSONMinified()
           if (li.rssFeed) {
             oldLibraryItem.rssFeed = Database.feedModel.getOldFeed(li.rssFeed).toJSONMinified()
@@ -93,6 +96,9 @@ module.exports = {
           if (li.size && !oldLibraryItem.media.size) {
             oldLibraryItem.media.size = li.size
           }
+          if (li.mediaItemShare) {
+            oldLibraryItem.mediaItemShare = li.mediaItemShare
+          }
           return oldLibraryItem
         }),
         count
@@ -100,7 +106,7 @@ module.exports = {
     } else {
       const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredLibraryItems(library.id, user, 'recent', null, 'addedAt', true, include, limit, 0)
       return {
-        libraryItems: libraryItems.map(li => {
+        libraryItems: libraryItems.map((li) => {
           const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(li).toJSONMinified()
           if (li.rssFeed) {
             oldLibraryItem.rssFeed = Database.feedModel.getOldFeed(li.rssFeed).toJSONMinified()
@@ -120,16 +126,16 @@ module.exports = {
 
   /**
    * Get library items for continue series shelf
-   * @param {import('../../objects/Library')} library 
-   * @param {oldUser} user 
-   * @param {string[]} include 
-   * @param {number} limit 
+   * @param {import('../../models/Library')} library
+   * @param {import('../../models/User')} user
+   * @param {string[]} include
+   * @param {number} limit
    * @returns {object} { libraryItems:LibraryItem[], count:number }
    */
   async getLibraryItemsContinueSeries(library, user, include, limit) {
     const { libraryItems, count } = await libraryItemsBookFilters.getContinueSeriesLibraryItems(library, user, include, limit, 0)
     return {
-      libraryItems: libraryItems.map(li => {
+      libraryItems: libraryItems.map((li) => {
         const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(li).toJSONMinified()
         if (li.rssFeed) {
           oldLibraryItem.rssFeed = Database.feedModel.getOldFeed(li.rssFeed).toJSONMinified()
@@ -137,6 +143,9 @@ module.exports = {
         if (li.series) {
           oldLibraryItem.media.metadata.series = li.series
         }
+        if (li.mediaItemShare) {
+          oldLibraryItem.mediaItemShare = li.mediaItemShare
+        }
         return oldLibraryItem
       }),
       count
@@ -145,21 +154,25 @@ module.exports = {
 
   /**
    * Get library items or podcast episodes for the "Listen Again" and "Read Again" shelf
-   * @param {import('../../objects/Library')} library 
-   * @param {oldUser} user 
-   * @param {string[]} include 
-   * @param {number} limit 
-   * @returns {object} { items:object[], count:number }
+   *
+   * @param {import('../../models/Library')} library
+   * @param {import('../../models/User')} user
+   * @param {string[]} include
+   * @param {number} limit
+   * @returns {Promise<{ items:oldLibraryItem[], count:number }>}
    */
   async getMediaFinished(library, user, include, limit) {
-    if (library.mediaType === 'book') {
+    if (library.isBook) {
       const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(library.id, user, 'progress', 'finished', 'progress', true, false, include, limit, 0)
       return {
-        items: libraryItems.map(li => {
+        items: libraryItems.map((li) => {
           const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(li).toJSONMinified()
           if (li.rssFeed) {
             oldLibraryItem.rssFeed = Database.feedModel.getOldFeed(li.rssFeed).toJSONMinified()
           }
+          if (li.mediaItemShare) {
+            oldLibraryItem.mediaItemShare = li.mediaItemShare
+          }
           return oldLibraryItem
         }),
         count
@@ -168,7 +181,7 @@ module.exports = {
       const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'progress', 'finished', 'progress', true, limit, 0)
       return {
         count,
-        items: libraryItems.map(li => {
+        items: libraryItems.map((li) => {
           const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(li).toJSONMinified()
           oldLibraryItem.recentEpisode = li.recentEpisode
           return oldLibraryItem
@@ -179,11 +192,11 @@ module.exports = {
 
   /**
    * Get series for recent series shelf
-   * @param {import('../../objects/Library')} library 
-   * @param {import('../../objects/user/User')} user
-   * @param {string[]} include 
-   * @param {number} limit 
-   * @returns {{ series:import('../../objects/entities/Series')[], count:number}} 
+   * @param {import('../../models/Library')} library
+   * @param {import('../../models/User')} user
+   * @param {string[]} include
+   * @param {number} limit
+   * @returns {{ series:any[], count:number}}
    */
   async getSeriesMostRecentlyAdded(library, user, include, limit) {
     if (!library.isBook) return { series: [], count: 0 }
@@ -201,7 +214,7 @@ module.exports = {
       {
         libraryId: library.id,
         createdAt: {
-          [Sequelize.Op.gte]: new Date(new Date() - (60 * 24 * 60 * 60 * 1000)) // 60 days ago
+          [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
         }
       }
     ]
@@ -209,9 +222,11 @@ module.exports = {
     // Handle library setting to hide single book series
     // TODO: Merge with existing query
     if (library.settings.hideSingleBookSeries) {
-      seriesWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id)`), {
-        [Sequelize.Op.gt]: 1
-      }))
+      seriesWhere.push(
+        Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id)`), {
+          [Sequelize.Op.gt]: 1
+        })
+      )
     }
 
     // Handle user permissions to only include series with at least 1 book
@@ -221,16 +236,18 @@ module.exports = {
       if (!user.canAccessExplicitContent) {
         attrQuery += ' AND b.explicit = 0'
       }
-      if (!user.permissions.accessAllTags && user.itemTagsSelected.length) {
+      if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
         if (user.permissions.selectedTagsNotAccessible) {
           attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0'
         } else {
           attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) > 0'
         }
       }
-      seriesWhere.push(Sequelize.where(Sequelize.literal(`(${attrQuery})`), {
-        [Sequelize.Op.gt]: 0
-      }))
+      seriesWhere.push(
+        Sequelize.where(Sequelize.literal(`(${attrQuery})`), {
+          [Sequelize.Op.gt]: 0
+        })
+      )
     }
 
     const { rows: series, count } = await Database.seriesModel.findAndCountAll({
@@ -254,14 +271,12 @@ module.exports = {
         },
         ...seriesIncludes
       ],
-      order: [
-        ['createdAt', 'DESC']
-      ]
+      order: [['createdAt', 'DESC']]
     })
 
     const allOldSeries = []
     for (const s of series) {
-      const oldSeries = s.getOldSeries().toJSON()
+      const oldSeries = s.toOldJSON()
 
       if (s.feeds?.length) {
         oldSeries.rssFeed = Database.feedModel.getOldFeed(s.feeds[0]).toJSONMinified()
@@ -276,18 +291,20 @@ module.exports = {
           sensitivity: 'base'
         })
       })
-      oldSeries.books = s.bookSeries.map(bs => {
-        const libraryItem = bs.book.libraryItem?.toJSON()
-        if (!libraryItem) {
-          Logger.warn(`Book series book has no libraryItem`, bs, bs.book, 'series=', series)
-          return null
-        }
+      oldSeries.books = s.bookSeries
+        .map((bs) => {
+          const libraryItem = bs.book.libraryItem?.toJSON()
+          if (!libraryItem) {
+            Logger.warn(`Book series book has no libraryItem`, bs, bs.book, 'series=', series)
+            return null
+          }
 
-        delete bs.book.libraryItem
-        libraryItem.media = bs.book
-        const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem).toJSONMinified()
-        return oldLibraryItem
-      }).filter(b => b)
+          delete bs.book.libraryItem
+          libraryItem.media = bs.book
+          const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem).toJSONMinified()
+          return oldLibraryItem
+        })
+        .filter((b) => b)
       allOldSeries.push(oldSeries)
     }
 
@@ -300,10 +317,11 @@ module.exports = {
   /**
    * Get most recently created authors for "Newest Authors" shelf
    * Author must be linked to at least 1 book
-   * @param {oldLibrary} library 
-   * @param {oldUser} user
-   * @param {number} limit 
-   * @returns {object} { authors:oldAuthor[], count:number }
+   *
+   * @param {import('../../models/Library')} library
+   * @param {import('../../models/User')} user
+   * @param {number} limit
+   * @returns {Promise<{ authors:oldAuthor[], count:number }>}
    */
   async getNewestAuthors(library, user, limit) {
     if (library.mediaType !== 'book') return { authors: [], count: 0 }
@@ -314,7 +332,7 @@ module.exports = {
       where: {
         libraryId: library.id,
         createdAt: {
-          [Sequelize.Op.gte]: new Date(new Date() - (60 * 24 * 60 * 60 * 1000)) // 60 days ago
+          [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
         }
       },
       replacements,
@@ -329,15 +347,13 @@ module.exports = {
       },
       limit,
       distinct: true,
-      order: [
-        ['createdAt', 'DESC']
-      ]
+      order: [['createdAt', 'DESC']]
     })
 
     return {
       authors: authors.map((au) => {
         const numBooks = au.books.length || 0
-        return au.getOldAuthor().toJSONExpanded(numBooks)
+        return au.toOldJSONExpanded(numBooks)
       }),
       count
     }
@@ -345,22 +361,25 @@ module.exports = {
 
   /**
    * Get book library items for the "Discover" shelf
-   * @param {oldLibrary} library 
-   * @param {oldUser} user 
-   * @param {string[]} include 
-   * @param {number} limit 
-   * @returns {object} {libraryItems:oldLibraryItem[], count:number}
+   * @param {import('../../models/Library')} library
+   * @param {import('../../models/User')} user
+   * @param {string[]} include
+   * @param {number} limit
+   * @returns {Promise<{libraryItems:oldLibraryItem[], count:number}>}
    */
   async getLibraryItemsToDiscover(library, user, include, limit) {
     if (library.mediaType !== 'book') return { libraryItems: [], count: 0 }
 
     const { libraryItems, count } = await libraryItemsBookFilters.getDiscoverLibraryItems(library.id, user, include, limit)
     return {
-      libraryItems: libraryItems.map(li => {
+      libraryItems: libraryItems.map((li) => {
         const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(li).toJSONMinified()
         if (li.rssFeed) {
           oldLibraryItem.rssFeed = Database.feedModel.getOldFeed(li.rssFeed).toJSONMinified()
         }
+        if (li.mediaItemShare) {
+          oldLibraryItem.mediaItemShare = li.mediaItemShare
+        }
         return oldLibraryItem
       }),
       count
@@ -369,10 +388,10 @@ module.exports = {
 
   /**
    * Get podcast episodes most recently added
-   * @param {oldLibrary} library 
-   * @param {oldUser} user 
-   * @param {number} limit 
-   * @returns {object} {libraryItems:oldLibraryItem[], count:number}
+   * @param {import('../../models/Library')} library
+   * @param {import('../../models/User')} user
+   * @param {number} limit
+   * @returns {Promise<{libraryItems:oldLibraryItem[], count:number}>}
    */
   async getNewestPodcastEpisodes(library, user, limit) {
     if (library.mediaType !== 'podcast') return { libraryItems: [], count: 0 }
@@ -380,7 +399,7 @@ module.exports = {
     const { libraryItems, count } = await libraryItemsPodcastFilters.getFilteredPodcastEpisodes(library.id, user, 'recent', null, 'createdAt', true, limit, 0)
     return {
       count,
-      libraryItems: libraryItems.map(li => {
+      libraryItems: libraryItems.map((li) => {
         const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(li).toJSONMinified()
         oldLibraryItem.recentEpisode = li.recentEpisode
         return oldLibraryItem
@@ -390,11 +409,11 @@ module.exports = {
 
   /**
    * Get library items for an author, optional use user permissions
-   * @param {oldAuthor} author 
-   * @param {[oldUser]} user 
-   * @param {number} limit 
-   * @param {number} offset 
-   * @returns {Promise<object>} { libraryItems:LibraryItem[], count:number }
+   * @param {import('../../models/Author')} author
+   * @param {import('../../models/User')} user
+   * @param {number} limit
+   * @param {number} offset
+   * @returns {Promise<{ libraryItems:import('../../objects/LibraryItem')[], count:number }>}
    */
   async getLibraryItemsForAuthor(author, user, limit, offset) {
     const { libraryItems, count } = await libraryItemsBookFilters.getFilteredLibraryItems(author.libraryId, user, 'authors', author.id, 'addedAt', true, false, [], limit, offset)
@@ -406,8 +425,8 @@ module.exports = {
 
   /**
    * Get book library items in a collection
-   * @param {oldCollection} collection 
-   * @returns {Promise<LibraryItem[]>}
+   * @param {oldCollection} collection
+   * @returns {Promise<import('../../models/LibraryItem')[]>}
    */
   getLibraryItemsForCollection(collection) {
     return libraryItemsBookFilters.getLibraryItemsForCollection(collection)
@@ -415,7 +434,7 @@ module.exports = {
 
   /**
    * Get filter data used in filter menus
-   * @param {string} mediaType 
+   * @param {string} mediaType
    * @param {string} libraryId
    * @returns {Promise<object>}
    */
@@ -439,6 +458,7 @@ module.exports = {
       narrators: new Set(),
       languages: new Set(),
       publishers: new Set(),
+      publishedDecades: new Set(),
       numIssues: 0
     }
 
@@ -473,7 +493,7 @@ module.exports = {
             libraryId: libraryId
           }
         },
-        attributes: ['tags', 'genres', 'publisher', 'narrators', 'language']
+        attributes: ['tags', 'genres', 'publisher', 'publishedYear', 'narrators', 'language']
       })
       for (const book of books) {
         if (book.libraryItem.isMissing || book.libraryItem.isInvalid) data.numIssues++
@@ -487,6 +507,11 @@ module.exports = {
           book.narrators.forEach((narrator) => data.narrators.add(narrator))
         }
         if (book.publisher) data.publishers.add(book.publisher)
+        // Check if published year exists and is valid
+        if (book.publishedYear && !isNaN(book.publishedYear) && book.publishedYear > 0 && book.publishedYear < 3000) {
+          const decade = (Math.floor(book.publishedYear / 10) * 10).toString()
+          data.publishedDecades.add(decade)
+        }
         if (book.language) data.languages.add(book.language)
       }
 
@@ -507,12 +532,13 @@ module.exports = {
       authors.forEach((a) => data.authors.push({ id: a.id, name: a.name }))
     }
 
-    data.authors = naturalSort(data.authors).asc(au => au.name)
+    data.authors = naturalSort(data.authors).asc((au) => au.name)
     data.genres = naturalSort([...data.genres]).asc()
     data.tags = naturalSort([...data.tags]).asc()
-    data.series = naturalSort(data.series).asc(se => se.name)
+    data.series = naturalSort(data.series).asc((se) => se.name)
     data.narrators = naturalSort([...data.narrators]).asc()
     data.publishers = naturalSort([...data.publishers]).asc()
+    data.publishedDecades = naturalSort([...data.publishedDecades]).asc()
     data.languages = naturalSort([...data.languages]).asc()
     data.loadedAt = Date.now()
     Database.libraryFilterData[libraryId] = data

server/utils/queries/libraryItemFilters.js

@@ -6,7 +6,7 @@ const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters')
 module.exports = {
   /**
    * Get all library items that have tags
-   * @param {string[]} tags 
+   * @param {string[]} tags
    * @returns {Promise<import('../../models/LibraryItem')[]>}
    */
   async getAllLibraryItemsWithTags(tags) {
@@ -71,7 +71,7 @@ module.exports = {
 
   /**
    * Get all library items that have genres
-   * @param {string[]} genres 
+   * @param {string[]} genres
    * @returns {Promise<import('../../models/LibraryItem')[]>}
    */
   async getAllLibraryItemsWithGenres(genres) {
@@ -131,10 +131,10 @@ module.exports = {
   },
 
   /**
- * Get all library items that have narrators
- * @param {string[]} narrators 
- * @returns {Promise<import('../../models/LibraryItem')[]>}
- */
+   * Get all library items that have narrators
+   * @param {string[]} narrators
+   * @returns {Promise<import('../../models/LibraryItem')[]>}
+   */
   async getAllLibraryItemsWithNarrators(narrators) {
     const libraryItems = []
     const booksWithGenre = await Database.bookModel.findAll({
@@ -172,24 +172,24 @@ module.exports = {
 
   /**
    * Search library items
-   * @param {import('../../objects/user/User')} oldUser 
-   * @param {import('../../objects/Library')} oldLibrary 
+   * @param {import('../../models/User')} user
+   * @param {import('../../models/Library')} library
    * @param {string} query
-   * @param {number} limit 
+   * @param {number} limit
    * @returns {{book:object[], narrators:object[], authors:object[], tags:object[], series:object[], podcast:object[]}}
    */
-  search(oldUser, oldLibrary, query, limit) {
-    if (oldLibrary.isBook) {
-      return libraryItemsBookFilters.search(oldUser, oldLibrary, query, limit, 0)
+  search(user, library, query, limit) {
+    if (library.isBook) {
+      return libraryItemsBookFilters.search(user, library, query, limit, 0)
     } else {
-      return libraryItemsPodcastFilters.search(oldUser, oldLibrary, query, limit, 0)
+      return libraryItemsPodcastFilters.search(user, library, query, limit, 0)
     }
   },
 
   /**
    * Get largest items in library
-   * @param {string} libraryId 
-   * @param {number} limit 
+   * @param {string} libraryId
+   * @param {number} limit
    * @returns {Promise<{ id:string, title:string, size:number }[]>}
    */
   async getLargestItems(libraryId, limit) {
@@ -208,12 +208,10 @@ module.exports = {
           attributes: ['id', 'title']
         }
       ],
-      order: [
-        ['size', 'DESC']
-      ],
+      order: [['size', 'DESC']],
       limit
     })
-    return libraryItems.map(libraryItem => {
+    return libraryItems.map((libraryItem) => {
       return {
         id: libraryItem.id,
         title: libraryItem.media.title,
@@ -221,4 +219,4 @@ module.exports = {
       }
     })
   }
-}
+}

server/utils/queries/libraryItemsBookFilters.js

@@ -2,12 +2,13 @@ const Sequelize = require('sequelize')
 const Database = require('../../Database')
 const Logger = require('../../Logger')
 const authorFilters = require('./authorFilters')
-const { asciiOnlyToLowerCase } = require('../index')
+
+const ShareManager = require('../../managers/ShareManager')
 
 module.exports = {
   /**
    * User permissions to restrict books for explicit content & tags
-   * @param {import('../../objects/user/User')} user
+   * @param {import('../../models/User')} user
    * @returns {{ bookWhere:Sequelize.WhereOptions, replacements:object }}
    */
   getUserPermissionBookWhereQuery(user) {
@@ -20,8 +21,8 @@ module.exports = {
         explicit: false
       })
     }
-    if (!user.permissions.accessAllTags && user.itemTagsSelected.length) {
-      replacements['userTagsSelected'] = user.itemTagsSelected
+    if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
+      replacements['userTagsSelected'] = user.permissions.itemTagsSelected
       if (user.permissions.selectedTagsNotAccessible) {
         bookWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0))
       } else {
@@ -218,7 +219,7 @@ module.exports = {
         mediaWhere[key] = {
           [Sequelize.Op.or]: [null, '']
         }
-      } else if (['genres', 'tags', 'narrators'].includes(value)) {
+      } else if (['genres', 'tags', 'narrators', 'chapters'].includes(value)) {
         mediaWhere[value] = {
           [Sequelize.Op.or]: [null, Sequelize.where(Sequelize.fn('json_array_length', Sequelize.col(value)), 0)]
         }
@@ -227,6 +228,12 @@ module.exports = {
       } else if (value === 'series') {
         mediaWhere['$series.id$'] = null
       }
+    } else if (group === 'publishedDecades') {
+      const startYear = parseInt(value)
+      const endYear = parseInt(value, 10) + 9
+      mediaWhere = Sequelize.where(Sequelize.literal('CAST(`book`.`publishedYear` AS INTEGER)'), {
+        [Sequelize.Op.between]: [startYear, endYear]
+      })
     }
 
     return { mediaWhere, replacements }
@@ -272,6 +279,8 @@ module.exports = {
       return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]]
     } else if (sortBy === 'progress') {
       return [[Sequelize.literal('mediaProgresses.updatedAt'), dir]]
+    } else if (sortBy === 'random') {
+      return [Database.sequelize.random()]
     }
     return []
   },
@@ -330,9 +339,9 @@ module.exports = {
   /**
    * Get library items for book media type using filter and sort
    * @param {string} libraryId
-   * @param {[oldUser]} user
-   * @param {[string]} filterGroup
-   * @param {[string]} filterValue
+   * @param {import('../../models/User')} user
+   * @param {string|null} filterGroup
+   * @param {string|null} filterValue
    * @param {string} sortBy
    * @param {string} sortDesc
    * @param {boolean} collapseseries
@@ -354,6 +363,7 @@ module.exports = {
       sortBy = 'media.metadata.title'
     }
     const includeRSSFeed = include.includes('rssfeed')
+    const includeMediaItemShare = !!user?.isAdminOrUp && include.includes('share')
 
     // For sorting by author name an additional attribute must be added
     //   with author names concatenated
@@ -409,6 +419,11 @@ module.exports = {
         model: Database.feedModel,
         required: true
       })
+    } else if (filterGroup === 'share-open') {
+      bookIncludes.push({
+        model: Database.mediaItemShareModel,
+        required: true
+      })
     } else if (filterGroup === 'ebooks' && filterValue === 'supplementary') {
       // TODO: Temp workaround for filtering supplementary ebook
       libraryItemWhere['libraryFiles'] = {
@@ -490,7 +505,6 @@ module.exports = {
     }
 
     let { mediaWhere, replacements } = this.getMediaGroupQuery(filterGroup, filterValue)
-
     let bookWhere = Array.isArray(mediaWhere) ? mediaWhere : [mediaWhere]
 
     // User permissions
@@ -605,6 +619,10 @@ module.exports = {
         libraryItem.rssFeed = libraryItem.feeds[0]
       }
 
+      if (includeMediaItemShare) {
+        libraryItem.mediaItemShare = ShareManager.findByMediaItemId(libraryItem.mediaId)
+      }
+
       libraryItem.media = book
 
       return libraryItem
@@ -623,8 +641,8 @@ module.exports = {
    * 2. Has no books in progress
    * 3. Has at least 1 unfinished book
    * TODO: Reduce queries
-   * @param {import('../../objects/Library')} library
-   * @param {import('../../objects/user/User')} user
+   * @param {import('../../models/Library')} library
+   * @param {import('../../models/User')} user
    * @param {string[]} include
    * @param {number} limit
    * @param {number} offset
@@ -659,7 +677,7 @@ module.exports = {
       where: [
         {
           id: {
-            [Sequelize.Op.notIn]: user.seriesHideFromContinueListening
+            [Sequelize.Op.notIn]: user.extraData?.seriesHideFromContinueListening || []
           },
           libraryId
         },
@@ -767,7 +785,7 @@ module.exports = {
    * Random selection of books that are not started
    *  - only includes the first book of a not-started series
    * @param {string} libraryId
-   * @param {oldUser} user
+   * @param {import('../../models/User')} user
    * @param {string[]} include
    * @param {number} limit
    * @returns {object} {libraryItems:LibraryItem, count:number}
@@ -898,7 +916,7 @@ module.exports = {
   /**
    * Get book library items in a collection
    * @param {oldCollection} collection
-   * @returns {Promise<LibraryItem[]>}
+   * @returns {Promise<import('../../models/LibraryItem')[]>}
    */
   async getLibraryItemsForCollection(collection) {
     if (!collection?.books?.length) {
@@ -941,42 +959,39 @@ module.exports = {
 
   /**
    * Get library items for series
-   * @param {import('../../objects/entities/Series')} oldSeries
-   * @param {import('../../objects/user/User')} [oldUser]
+   * @param {import('../../models/Series')} series
+   * @param {import('../../models/User')} [user]
    * @returns {Promise<import('../../objects/LibraryItem')[]>}
    */
-  async getLibraryItemsForSeries(oldSeries, oldUser) {
-    const { libraryItems } = await this.getFilteredLibraryItems(oldSeries.libraryId, oldUser, 'series', oldSeries.id, null, null, false, [], null, null)
+  async getLibraryItemsForSeries(series, user) {
+    const { libraryItems } = await this.getFilteredLibraryItems(series.libraryId, user, 'series', series.id, null, null, false, [], null, null)
     return libraryItems.map((li) => Database.libraryItemModel.getOldLibraryItem(li))
   },
 
   /**
    * Search books, authors, series
-   * @param {import('../../objects/user/User')} oldUser
-   * @param {import('../../objects/Library')} oldLibrary
+   * @param {import('../../models/User')} user
+   * @param {import('../../models/Library')} library
    * @param {string} query
    * @param {number} limit
    * @param {number} offset
    * @returns {{book:object[], narrators:object[], authors:object[], tags:object[], series:object[]}}
    */
-  async search(oldUser, oldLibrary, query, limit, offset) {
-    const userPermissionBookWhere = this.getUserPermissionBookWhereQuery(oldUser)
+  async search(user, library, query, limit, offset) {
+    const userPermissionBookWhere = this.getUserPermissionBookWhereQuery(user)
+
+    const textSearchQuery = await Database.createTextSearchQuery(query)
+
+    const matchTitle = textSearchQuery.matchExpression('title')
+    const matchSubtitle = textSearchQuery.matchExpression('subtitle')
 
     // Search title, subtitle, asin, isbn
     const books = await Database.bookModel.findAll({
       where: [
         {
           [Sequelize.Op.or]: [
-            {
-              title: {
-                [Sequelize.Op.substring]: query
-              }
-            },
-            {
-              subtitle: {
-                [Sequelize.Op.substring]: query
-              }
-            },
+            Sequelize.literal(matchTitle),
+            Sequelize.literal(matchSubtitle),
             {
               asin: {
                 [Sequelize.Op.substring]: query
@@ -996,7 +1011,7 @@ module.exports = {
         {
           model: Database.libraryItemModel,
           where: {
-            libraryId: oldLibrary.id
+            libraryId: library.id
           }
         },
         {
@@ -1026,33 +1041,18 @@ module.exports = {
       const libraryItem = book.libraryItem
       delete book.libraryItem
       libraryItem.media = book
-
-      let matchText = null
-      let matchKey = null
-      for (const key of ['title', 'subtitle', 'asin', 'isbn']) {
-        const valueToLower = asciiOnlyToLowerCase(book[key])
-        if (valueToLower.includes(query)) {
-          matchText = book[key]
-          matchKey = key
-          break
-        }
-      }
-
-      if (matchKey) {
-        itemMatches.push({
-          matchText,
-          matchKey,
-          libraryItem: Database.libraryItemModel.getOldLibraryItem(libraryItem).toJSONExpanded()
-        })
-      }
+      itemMatches.push({
+        libraryItem: Database.libraryItemModel.getOldLibraryItem(libraryItem).toJSONExpanded()
+      })
     }
 
+    const matchJsonValue = textSearchQuery.matchExpression('json_each.value')
+
     // Search narrators
     const narratorMatches = []
-    const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND json_each.value LIKE :query AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
+    const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
       replacements: {
-        query: `%${query}%`,
-        libraryId: oldLibrary.id,
+        libraryId: library.id,
         limit,
         offset
       },
@@ -1067,10 +1067,9 @@ module.exports = {
 
     // Search tags
     const tagMatches = []
-    const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND json_each.value LIKE :query AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
+    const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
       replacements: {
-        query: `%${query}%`,
-        libraryId: oldLibrary.id,
+        libraryId: library.id,
         limit,
         offset
       },
@@ -1083,13 +1082,33 @@ module.exports = {
       })
     }
 
+    // Search genres
+    const genreMatches = []
+    const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
+      replacements: {
+        libraryId: library.id,
+        limit,
+        offset
+      },
+      raw: true
+    })
+    for (const row of genreResults) {
+      genreMatches.push({
+        name: row.value,
+        numItems: row.numItems
+      })
+    }
+
     // Search series
+    const matchName = textSearchQuery.matchExpression('name')
     const allSeries = await Database.seriesModel.findAll({
       where: {
-        name: {
-          [Sequelize.Op.substring]: query
-        },
-        libraryId: oldLibrary.id
+        [Sequelize.Op.and]: [
+          Sequelize.literal(matchName),
+          {
+            libraryId: library.id
+          }
+        ]
       },
       replacements: userPermissionBookWhere.replacements,
       include: {
@@ -1116,18 +1135,19 @@ module.exports = {
         return Database.libraryItemModel.getOldLibraryItem(libraryItem).toJSON()
       })
       seriesMatches.push({
-        series: series.getOldSeries().toJSON(),
+        series: series.toOldJSON(),
         books
       })
     }
 
     // Search authors
-    const authorMatches = await authorFilters.search(oldLibrary.id, query, limit, offset)
+    const authorMatches = await authorFilters.search(library.id, textSearchQuery, limit, offset)
 
     return {
       book: itemMatches,
       narrators: narratorMatches,
       tags: tagMatches,
+      genres: genreMatches,
       series: seriesMatches,
       authors: authorMatches
     }

server/utils/queries/libraryItemsPodcastFilters.js

@@ -1,13 +1,11 @@
-
 const Sequelize = require('sequelize')
 const Database = require('../../Database')
 const Logger = require('../../Logger')
-const { asciiOnlyToLowerCase } = require('../index')
 
 module.exports = {
   /**
    * User permissions to restrict podcasts for explicit content & tags
-   * @param {import('../../objects/user/User')} user
+   * @param {import('../../models/User')} user
    * @returns {{ podcastWhere:Sequelize.WhereOptions, replacements:object }}
    */
   getUserPermissionPodcastWhereQuery(user) {
@@ -18,16 +16,20 @@ module.exports = {
         explicit: false
       })
     }
-    if (!user.permissions.accessAllTags && user.itemTagsSelected.length) {
-      replacements['userTagsSelected'] = user.itemTagsSelected
+
+    if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
+      replacements['userTagsSelected'] = user.permissions.itemTagsSelected
       if (user.permissions.selectedTagsNotAccessible) {
         podcastWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), 0))
       } else {
-        podcastWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), {
-          [Sequelize.Op.gte]: 1
-        }))
+        podcastWhere.push(
+          Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected))`), {
+            [Sequelize.Op.gte]: 1
+          })
+        )
       }
     }
+
     return {
       podcastWhere,
       replacements
@@ -88,6 +90,8 @@ module.exports = {
       }
     } else if (sortBy === 'media.numTracks') {
       return [['numEpisodes', dir]]
+    } else if (sortBy === 'random') {
+      return [Database.sequelize.random()]
     }
     return []
   },
@@ -95,7 +99,7 @@ module.exports = {
   /**
    * Get library items for podcast media type using filter and sort
    * @param {string} libraryId
-   * @param {oldUser} user
+   * @param {import('../../models/User')} user
    * @param {[string]} filterGroup
    * @param {[string]} filterValue
    * @param {string} sortBy
@@ -130,7 +134,7 @@ module.exports = {
       ]
     } else if (filterGroup === 'recent') {
       libraryItemWhere['createdAt'] = {
-        [Sequelize.Op.gte]: new Date(new Date() - (60 * 24 * 60 * 60 * 1000)) // 60 days ago
+        [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
       }
     }
 
@@ -154,10 +158,7 @@ module.exports = {
       replacements,
       distinct: true,
       attributes: {
-        include: [
-          [Sequelize.literal(`(SELECT count(*) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'numEpisodes'],
-          ...podcastIncludes
-        ]
+        include: [[Sequelize.literal(`(SELECT count(*) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'numEpisodes'], ...podcastIncludes]
       },
       include: [
         {
@@ -200,7 +201,7 @@ module.exports = {
   /**
    * Get podcast episodes filtered and sorted
    * @param {string} libraryId
-   * @param {oldUser} user
+   * @param {import('../../models/User')} user
    * @param {[string]} filterGroup
    * @param {[string]} filterValue
    * @param {string} sortBy
@@ -251,7 +252,7 @@ module.exports = {
       }
     } else if (filterGroup === 'recent') {
       podcastEpisodeWhere['createdAt'] = {
-        [Sequelize.Op.gte]: new Date(new Date() - (60 * 24 * 60 * 60 * 1000)) // 60 days ago
+        [Sequelize.Op.gte]: new Date(new Date() - 60 * 24 * 60 * 60 * 1000) // 60 days ago
       }
     }
 
@@ -304,30 +305,28 @@ module.exports = {
 
   /**
    * Search podcasts
-   * @param {import('../../objects/user/User')} oldUser
-   * @param {import('../../objects/Library')} oldLibrary
+   * @param {import('../../models/User')} user
+   * @param {import('../../models/Library')} library
    * @param {string} query
    * @param {number} limit
    * @param {number} offset
    * @returns {{podcast:object[], tags:object[]}}
    */
-  async search(oldUser, oldLibrary, query, limit, offset) {
-    const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(oldUser)
+  async search(user, library, query, limit, offset) {
+    const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(user)
+
+    const textSearchQuery = await Database.createTextSearchQuery(query)
+
+    const matchTitle = textSearchQuery.matchExpression('title')
+    const matchAuthor = textSearchQuery.matchExpression('author')
+
     // Search title, author, itunesId, itunesArtistId
     const podcasts = await Database.podcastModel.findAll({
       where: [
         {
           [Sequelize.Op.or]: [
-            {
-              title: {
-                [Sequelize.Op.substring]: query
-              }
-            },
-            {
-              author: {
-                [Sequelize.Op.substring]: query
-              }
-            },
+            Sequelize.literal(matchTitle),
+            Sequelize.literal(matchAuthor),
             {
               itunesId: {
                 [Sequelize.Op.substring]: query
@@ -347,7 +346,7 @@ module.exports = {
         {
           model: Database.libraryItemModel,
           where: {
-            libraryId: oldLibrary.id
+            libraryId: library.id
           }
         }
       ],
@@ -363,33 +362,18 @@ module.exports = {
       const libraryItem = podcast.libraryItem
       delete podcast.libraryItem
       libraryItem.media = podcast
-
-      let matchText = null
-      let matchKey = null
-      for (const key of ['title', 'author', 'itunesId', 'itunesArtistId']) {
-        const valueToLower = asciiOnlyToLowerCase(podcast[key])
-        if (valueToLower.includes(query)) {
-          matchText = podcast[key]
-          matchKey = key
-          break
-        }
-      }
-
-      if (matchKey) {
-        itemMatches.push({
-          matchText,
-          matchKey,
-          libraryItem: Database.libraryItemModel.getOldLibraryItem(libraryItem).toJSONExpanded()
-        })
-      }
+      itemMatches.push({
+        libraryItem: Database.libraryItemModel.getOldLibraryItem(libraryItem).toJSONExpanded()
+      })
     }
 
+    const matchJsonValue = textSearchQuery.matchExpression('json_each.value')
+
     // Search tags
     const tagMatches = []
-    const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND json_each.value LIKE :query AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
+    const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
       replacements: {
-        query: `%${query}%`,
-        libraryId: oldLibrary.id,
+        libraryId: library.id,
         limit,
         offset
       },
@@ -402,22 +386,40 @@ module.exports = {
       })
     }
 
+    // Search genres
+    const genreMatches = []
+    const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
+      replacements: {
+        libraryId: library.id,
+        limit,
+        offset
+      },
+      raw: true
+    })
+    for (const row of genreResults) {
+      genreMatches.push({
+        name: row.value,
+        numItems: row.numItems
+      })
+    }
+
     return {
       podcast: itemMatches,
-      tags: tagMatches
+      tags: tagMatches,
+      genres: genreMatches
     }
   },
 
   /**
    * Most recent podcast episodes not finished
-   * @param {import('../../objects/user/User')} oldUser
-   * @param {import('../../objects/Library')} oldLibrary
+   * @param {import('../../models/User')} user
+   * @param {import('../../models/Library')} library
    * @param {number} limit
    * @param {number} offset
    * @returns {Promise<object[]>}
    */
-  async getRecentEpisodes(oldUser, oldLibrary, limit, offset) {
-    const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(oldUser)
+  async getRecentEpisodes(user, library, limit, offset) {
+    const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(user)
 
     const episodes = await Database.podcastEpisodeModel.findAll({
       where: {
@@ -434,21 +436,19 @@ module.exports = {
           include: {
             model: Database.libraryItemModel,
             where: {
-              libraryId: oldLibrary.id
+              libraryId: library.id
             }
           }
         },
         {
           model: Database.mediaProgressModel,
           where: {
-            userId: oldUser.id
+            userId: user.id
           },
           required: false
         }
       ],
-      order: [
-        ['publishedAt', 'DESC']
-      ],
+      order: [['publishedAt', 'DESC']],
       subQuery: false,
       limit,
       offset
@@ -512,18 +512,14 @@ module.exports = {
   },
 
   /**
-   * Get the longest podcasts in library
+   * Get longest podcasts in library
    * @param {string} libraryId
    * @param {number} limit
    * @returns {Promise<{ id:string, title:string, duration:number }[]>}
    */
   async getLongestPodcasts(libraryId, limit) {
     const podcasts = await Database.podcastModel.findAll({
-      attributes: [
-        'id',
-        'title',
-        [Sequelize.literal(`(SELECT SUM(json_extract(pe.audioFile, '$.duration')) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']
-      ],
+      attributes: ['id', 'title', [Sequelize.literal(`(SELECT SUM(json_extract(pe.audioFile, '$.duration')) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'duration']],
       include: {
         model: Database.libraryItemModel,
         attributes: ['id', 'libraryId'],
@@ -531,12 +527,10 @@ module.exports = {
           libraryId
         }
       },
-      order: [
-        ['duration', 'DESC']
-      ],
+      order: [['duration', 'DESC']],
       limit
     })
-    return podcasts.map(podcast => {
+    return podcasts.map((podcast) => {
       return {
         id: podcast.libraryItem.id,
         title: podcast.title,

server/utils/queries/seriesFilters.js

@@ -10,15 +10,15 @@ module.exports = {
 
   /**
    * Get series filtered and sorted
-   * 
-   * @param {import('../../objects/Library')} library 
-   * @param {import('../../objects/user/User')} user 
-   * @param {string} filterBy 
-   * @param {string} sortBy 
-   * @param {boolean} sortDesc 
-   * @param {string[]} include 
-   * @param {number} limit 
-   * @param {number} offset 
+   *
+   * @param {import('../../models/Library')} library
+   * @param {import('../../models/User')} user
+   * @param {string} filterBy
+   * @param {string} sortBy
+   * @param {boolean} sortDesc
+   * @param {string[]} include
+   * @param {number} limit
+   * @param {number} offset
    * @returns {Promise<{ series:object[], count:number }>}
    */
   async getFilteredSeries(library, user, filterBy, sortBy, sortDesc, include, limit, offset) {
@@ -26,7 +26,7 @@ module.exports = {
     let filterGroup = null
     if (filterBy) {
       const searchGroups = ['genres', 'tags', 'authors', 'progress', 'narrators', 'publishers', 'languages']
-      const group = searchGroups.find(_group => filterBy.startsWith(_group + '.'))
+      const group = searchGroups.find((_group) => filterBy.startsWith(_group + '.'))
       filterGroup = group || filterBy
       filterValue = group ? this.decode(filterBy.replace(`${group}.`, '')) : null
     }
@@ -49,9 +49,11 @@ module.exports = {
     // Handle library setting to hide single book series
     // TODO: Merge with existing query
     if (library.settings.hideSingleBookSeries) {
-      seriesWhere.push(Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id)`), {
-        [Sequelize.Op.gt]: 1
-      }))
+      seriesWhere.push(
+        Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND bs.bookId = b.id)`), {
+          [Sequelize.Op.gt]: 1
+        })
+      )
     }
 
     // Handle filters
@@ -91,7 +93,7 @@ module.exports = {
       if (!user.canAccessExplicitContent) {
         attrQuery += ' AND b.explicit = 0'
       }
-      if (!user.permissions.accessAllTags && user.itemTagsSelected.length) {
+      if (!user.permissions?.accessAllTags && user.permissions?.itemTagsSelected?.length) {
         if (user.permissions.selectedTagsNotAccessible) {
           attrQuery += ' AND (SELECT count(*) FROM json_each(tags) WHERE json_valid(tags) AND json_each.value IN (:userTagsSelected)) = 0'
         } else {
@@ -101,9 +103,11 @@ module.exports = {
     }
 
     if (attrQuery) {
-      seriesWhere.push(Sequelize.where(Sequelize.literal(`(${attrQuery})`), {
-        [Sequelize.Op.gt]: 0
-      }))
+      seriesWhere.push(
+        Sequelize.where(Sequelize.literal(`(${attrQuery})`), {
+          [Sequelize.Op.gt]: 0
+        })
+      )
     }
 
     const order = []
@@ -133,6 +137,8 @@ module.exports = {
     } else if (sortBy === 'lastBookUpdated') {
       seriesAttributes.include.push([Sequelize.literal('(SELECT MAX(b.updatedAt) FROM books b, bookSeries bs WHERE bs.seriesId = series.id AND b.id = bs.bookId)'), 'mostRecentBookUpdated'])
       order.push(['mostRecentBookUpdated', dir])
+    } else if (sortBy === 'random') {
+      order.push(Database.sequelize.random())
     }
 
     const { rows: series, count } = await Database.seriesModel.findAndCountAll({
@@ -165,7 +171,7 @@ module.exports = {
     // Map series to old series
     const allOldSeries = []
     for (const s of series) {
-      const oldSeries = s.getOldSeries().toJSON()
+      const oldSeries = s.toOldJSON()
 
       if (s.dataValues.totalDuration) {
         oldSeries.totalDuration = s.dataValues.totalDuration
@@ -184,7 +190,7 @@ module.exports = {
           sensitivity: 'base'
         })
       })
-      oldSeries.books = s.bookSeries.map(bs => {
+      oldSeries.books = s.bookSeries.map((bs) => {
         const libraryItem = bs.book.libraryItem.toJSON()
         delete bs.book.libraryItem
         libraryItem.media = bs.book

server/utils/queries/userStats.js

@@ -6,8 +6,8 @@ const fsExtra = require('../../libs/fsExtra')
 
 module.exports = {
   /**
-   * 
-   * @param {string} userId 
+   *
+   * @param {string} userId
    * @param {number} year YYYY
    * @returns {Promise<PlaybackSession[]>}
    */
@@ -35,8 +35,8 @@ module.exports = {
   },
 
   /**
-   * 
-   * @param {string} userId 
+   *
+   * @param {string} userId
    * @param {number} year YYYY
    * @returns {Promise<MediaProgress[]>}
    */
@@ -65,11 +65,10 @@ module.exports = {
   },
 
   /**
-   * @param {import('../../objects/user/User')} user
+   * @param {string} userId
    * @param {number} year YYYY
    */
-  async getStatsForYear(user, year) {
-    const userId = user.id
+  async getStatsForYear(userId, year) {
     const listeningSessions = await this.getUserListeningSessionsForYear(userId, year)
     const bookProgressesFinished = await this.getBookMediaProgressFinishedForYear(userId, year)
 
@@ -91,7 +90,7 @@ module.exports = {
     let longestAudiobookFinished = null
     for (const mediaProgress of bookProgressesFinished) {
       // Grab first 5 that have a cover
-      if (mediaProgress.mediaItem?.coverPath && !finishedBooksWithCovers.includes(mediaProgress.mediaItem.libraryItem.id) && finishedBooksWithCovers.length < 5 && await fsExtra.pathExists(mediaProgress.mediaItem.coverPath)) {
+      if (mediaProgress.mediaItem?.coverPath && !finishedBooksWithCovers.includes(mediaProgress.mediaItem.libraryItem.id) && finishedBooksWithCovers.length < 5 && (await fsExtra.pathExists(mediaProgress.mediaItem.coverPath))) {
         finishedBooksWithCovers.push(mediaProgress.mediaItem.libraryItem.id)
       }
 
@@ -108,7 +107,7 @@ module.exports = {
     // Get listening session stats
     for (const ls of listeningSessions) {
       // Grab first 25 that have a cover
-      if (ls.mediaItem?.coverPath && !booksWithCovers.includes(ls.mediaItem.libraryItem.id) && !finishedBooksWithCovers.includes(ls.mediaItem.libraryItem.id) && booksWithCovers.length < 25 && await fsExtra.pathExists(ls.mediaItem.coverPath)) {
+      if (ls.mediaItem?.coverPath && !booksWithCovers.includes(ls.mediaItem.libraryItem.id) && !finishedBooksWithCovers.includes(ls.mediaItem.libraryItem.id) && booksWithCovers.length < 25 && (await fsExtra.pathExists(ls.mediaItem.coverPath))) {
         booksWithCovers.push(ls.mediaItem.libraryItem.id)
       }
 
@@ -141,7 +140,7 @@ module.exports = {
         })
 
         // Filter out bad genres like "audiobook" and "audio book"
-        const genres = (ls.mediaMetadata.genres || []).filter(g => g && !g.toLowerCase().includes('audiobook') && !g.toLowerCase().includes('audio book'))
+        const genres = (ls.mediaMetadata.genres || []).filter((g) => g && !g.toLowerCase().includes('audiobook') && !g.toLowerCase().includes('audio book'))
         genres.forEach((genre) => {
           if (!genreListeningMap[genre]) genreListeningMap[genre] = 0
           genreListeningMap[genre] += listeningSessionListeningTime
@@ -156,10 +155,13 @@ module.exports = {
     totalPodcastListeningTime = Math.round(totalPodcastListeningTime)
 
     let topAuthors = null
-    topAuthors = Object.keys(authorListeningMap).map(authorName => ({
-      name: authorName,
-      time: Math.round(authorListeningMap[authorName])
-    })).sort((a, b) => b.time - a.time).slice(0, 3)
+    topAuthors = Object.keys(authorListeningMap)
+      .map((authorName) => ({
+        name: authorName,
+        time: Math.round(authorListeningMap[authorName])
+      }))
+      .sort((a, b) => b.time - a.time)
+      .slice(0, 3)
 
     let mostListenedNarrator = null
     for (const narrator in narratorListeningMap) {
@@ -172,10 +174,13 @@ module.exports = {
     }
 
     let topGenres = null
-    topGenres = Object.keys(genreListeningMap).map(genre => ({
-      genre,
-      time: Math.round(genreListeningMap[genre])
-    })).sort((a, b) => b.time - a.time).slice(0, 3)
+    topGenres = Object.keys(genreListeningMap)
+      .map((genre) => ({
+        genre,
+        time: Math.round(genreListeningMap[genre])
+      }))
+      .sort((a, b) => b.time - a.time)
+      .slice(0, 3)
 
     let mostListenedMonth = null
     for (const month in monthListeningMap) {

server/utils/scandir.js

@@ -19,8 +19,7 @@ const parseNameString = require('./parsers/parseNameString')
 function isMediaFile(mediaType, ext, audiobooksOnly = false) {
   if (!ext) return false
   const extclean = ext.slice(1).toLowerCase()
-  if (mediaType === 'podcast' || mediaType === 'music') return globals.SupportedAudioTypes.includes(extclean)
-  else if (mediaType === 'video') return globals.SupportedVideoTypes.includes(extclean)
+  if (mediaType === 'podcast') return globals.SupportedAudioTypes.includes(extclean)
   else if (audiobooksOnly) return globals.SupportedAudioTypes.includes(extclean)
   return globals.SupportedAudioTypes.includes(extclean) || globals.SupportedEbookTypes.includes(extclean)
 }
@@ -35,29 +34,33 @@ module.exports.checkFilepathIsAudioFile = checkFilepathIsAudioFile
 
 /**
  * TODO: Function needs to be re-done
- * @param {string} mediaType 
+ * @param {string} mediaType
  * @param {string[]} paths array of relative file paths
  * @returns {Record<string,string[]>} map of files grouped into potential libarary item dirs
  */
 function groupFilesIntoLibraryItemPaths(mediaType, paths) {
   // Step 1: Clean path, Remove leading "/", Filter out non-media files in root dir
   var nonMediaFilePaths = []
-  var pathsFiltered = paths.map(path => {
-    return path.startsWith('/') ? path.slice(1) : path
-  }).filter(path => {
-    let parsedPath = Path.parse(path)
-    // Is not in root dir OR is a book media file
-    if (parsedPath.dir) {
-      if (!isMediaFile(mediaType, parsedPath.ext, false)) { // Seperate out non-media files
-        nonMediaFilePaths.push(path)
-        return false
+  var pathsFiltered = paths
+    .map((path) => {
+      return path.startsWith('/') ? path.slice(1) : path
+    })
+    .filter((path) => {
+      let parsedPath = Path.parse(path)
+      // Is not in root dir OR is a book media file
+      if (parsedPath.dir) {
+        if (!isMediaFile(mediaType, parsedPath.ext, false)) {
+          // Seperate out non-media files
+          nonMediaFilePaths.push(path)
+          return false
+        }
+        return true
+      } else if (mediaType === 'book' && isMediaFile(mediaType, parsedPath.ext, false)) {
+        // (book media type supports single file audiobooks/ebooks in root dir)
+        return true
       }
-      return true
-    } else if (mediaType === 'book' && isMediaFile(mediaType, parsedPath.ext, false)) { // (book media type supports single file audiobooks/ebooks in root dir)
-      return true
-    }
-    return false
-  })
+      return false
+    })
 
   // Step 2: Sort by least number of directories
   pathsFiltered.sort((a, b) => {
@@ -69,7 +72,9 @@ function groupFilesIntoLibraryItemPaths(mediaType, paths) {
   // Step 3: Group files in dirs
   var itemGroup = {}
   pathsFiltered.forEach((path) => {
-    var dirparts = Path.dirname(path).split('/').filter(p => !!p && p !== '.') //  dirname returns . if no directory
+    var dirparts = Path.dirname(path)
+      .split('/')
+      .filter((p) => !!p && p !== '.') //  dirname returns . if no directory
     var numparts = dirparts.length
     var _path = ''
 
@@ -82,14 +87,17 @@ function groupFilesIntoLibraryItemPaths(mediaType, paths) {
         var dirpart = dirparts.shift()
         _path = Path.posix.join(_path, dirpart)
 
-        if (itemGroup[_path]) { // Directory already has files, add file
+        if (itemGroup[_path]) {
+          // Directory already has files, add file
           var relpath = Path.posix.join(dirparts.join('/'), Path.basename(path))
           itemGroup[_path].push(relpath)
           return
-        } else if (!dirparts.length) { // This is the last directory, create group
+        } else if (!dirparts.length) {
+          // This is the last directory, create group
           itemGroup[_path] = [Path.basename(path)]
           return
-        } else if (dirparts.length === 1 && /^cd\d{1,3}$/i.test(dirparts[0])) { // Next directory is the last and is a CD dir, create group
+        } else if (dirparts.length === 1 && /^cd\d{1,3}$/i.test(dirparts[0])) {
+          // Next directory is the last and is a CD dir, create group
           itemGroup[_path] = [Path.posix.join(dirparts[0], Path.basename(path))]
           return
         }
@@ -99,7 +107,6 @@ function groupFilesIntoLibraryItemPaths(mediaType, paths) {
 
   // Step 4: Add in non-media files if they fit into item group
   if (nonMediaFilePaths.length) {
-
     for (const nonMediaFilePath of nonMediaFilePaths) {
       const pathDir = Path.dirname(nonMediaFilePath)
       const filename = Path.basename(nonMediaFilePath)
@@ -111,7 +118,8 @@ function groupFilesIntoLibraryItemPaths(mediaType, paths) {
       for (let i = 0; i < numparts; i++) {
         const dirpart = dirparts.shift()
         _path = Path.posix.join(_path, dirpart)
-        if (itemGroup[_path]) { // Directory is a group
+        if (itemGroup[_path]) {
+          // Directory is a group
           const relpath = Path.posix.join(dirparts.join('/'), filename)
           itemGroup[_path].push(relpath)
         } else if (!dirparts.length) {
@@ -126,31 +134,22 @@ function groupFilesIntoLibraryItemPaths(mediaType, paths) {
 module.exports.groupFilesIntoLibraryItemPaths = groupFilesIntoLibraryItemPaths
 
 /**
- * @param {string} mediaType 
+ * @param {string} mediaType
  * @param {{name:string, path:string, dirpath:string, reldirpath:string, fullpath:string, extension:string, deep:number}[]} fileItems (see recurseFiles)
- * @param {boolean} [audiobooksOnly=false] 
+ * @param {boolean} [audiobooksOnly=false]
  * @returns {Record<string,string[]>} map of files grouped into potential libarary item dirs
  */
 function groupFileItemsIntoLibraryItemDirs(mediaType, fileItems, audiobooksOnly = false) {
-  // Handle music where every audio file is a library item
-  if (mediaType === 'music') {
-    const audioFileGroup = {}
-    fileItems.filter(i => isMediaFile(mediaType, i.extension, audiobooksOnly)).forEach((item) => {
-      audioFileGroup[item.path] = item.path
-    })
-    return audioFileGroup
-  }
-
   // Step 1: Filter out non-book-media files in root dir (with depth of 0)
-  const itemsFiltered = fileItems.filter(i => {
-    return i.deep > 0 || ((mediaType === 'book' || mediaType === 'video' || mediaType === 'music') && isMediaFile(mediaType, i.extension, audiobooksOnly))
+  const itemsFiltered = fileItems.filter((i) => {
+    return i.deep > 0 || (mediaType === 'book' && isMediaFile(mediaType, i.extension, audiobooksOnly))
   })
 
   // Step 2: Seperate media files and other files
   //     - Directories without a media file will not be included
   const mediaFileItems = []
   const otherFileItems = []
-  itemsFiltered.forEach(item => {
+  itemsFiltered.forEach((item) => {
     if (isMediaFile(mediaType, item.extension, audiobooksOnly)) mediaFileItems.push(item)
     else otherFileItems.push(item)
   })
@@ -158,7 +157,7 @@ function groupFileItemsIntoLibraryItemDirs(mediaType, fileItems, audiobooksOnly
   // Step 3: Group audio files in library items
   const libraryItemGroup = {}
   mediaFileItems.forEach((item) => {
-    const dirparts = item.reldirpath.split('/').filter(p => !!p)
+    const dirparts = item.reldirpath.split('/').filter((p) => !!p)
     const numparts = dirparts.length
     let _path = ''
 
@@ -171,14 +170,17 @@ function groupFileItemsIntoLibraryItemDirs(mediaType, fileItems, audiobooksOnly
         const dirpart = dirparts.shift()
         _path = Path.posix.join(_path, dirpart)
 
-        if (libraryItemGroup[_path]) { // Directory already has files, add file
+        if (libraryItemGroup[_path]) {
+          // Directory already has files, add file
           const relpath = Path.posix.join(dirparts.join('/'), item.name)
           libraryItemGroup[_path].push(relpath)
           return
-        } else if (!dirparts.length) { // This is the last directory, create group
+        } else if (!dirparts.length) {
+          // This is the last directory, create group
           libraryItemGroup[_path] = [item.name]
           return
-        } else if (dirparts.length === 1 && /^cd\d{1,3}$/i.test(dirparts[0])) { // Next directory is the last and is a CD dir, create group
+        } else if (dirparts.length === 1 && /^cd\d{1,3}$/i.test(dirparts[0])) {
+          // Next directory is the last and is a CD dir, create group
           libraryItemGroup[_path] = [Path.posix.join(dirparts[0], item.name)]
           return
         }
@@ -196,7 +198,8 @@ function groupFileItemsIntoLibraryItemDirs(mediaType, fileItems, audiobooksOnly
     for (let i = 0; i < numparts; i++) {
       const dirpart = dirparts.shift()
       _path = Path.posix.join(_path, dirpart)
-      if (libraryItemGroup[_path]) { // Directory is audiobook group
+      if (libraryItemGroup[_path]) {
+        // Directory is audiobook group
         const relpath = Path.posix.join(dirparts.join('/'), item.name)
         libraryItemGroup[_path].push(relpath)
         return
@@ -209,33 +212,35 @@ module.exports.groupFileItemsIntoLibraryItemDirs = groupFileItemsIntoLibraryItem
 
 /**
  * Get LibraryFile from filepath
- * @param {string} libraryItemPath 
- * @param {string[]} files 
+ * @param {string} libraryItemPath
+ * @param {string[]} files
  * @returns {import('../objects/files/LibraryFile')}
  */
 function buildLibraryFile(libraryItemPath, files) {
-  return Promise.all(files.map(async (file) => {
-    const filePath = Path.posix.join(libraryItemPath, file)
-    const newLibraryFile = new LibraryFile()
-    await newLibraryFile.setDataFromPath(filePath, file)
-    return newLibraryFile
-  }))
+  return Promise.all(
+    files.map(async (file) => {
+      const filePath = Path.posix.join(libraryItemPath, file)
+      const newLibraryFile = new LibraryFile()
+      await newLibraryFile.setDataFromPath(filePath, file)
+      return newLibraryFile
+    })
+  )
 }
 module.exports.buildLibraryFile = buildLibraryFile
 
 /**
  * Get details parsed from filenames
- * 
- * @param {string} relPath 
- * @param {boolean} parseSubtitle 
+ *
+ * @param {string} relPath
+ * @param {boolean} parseSubtitle
  * @returns {LibraryItemFilenameMetadata}
  */
 function getBookDataFromDir(relPath, parseSubtitle = false) {
   const splitDir = relPath.split('/')
 
   var folder = splitDir.pop() // Audio files will always be in the directory named for the title
-  series = (splitDir.length > 1) ? splitDir.pop() : null // If there are at least 2 more directories, next furthest will be the series
-  author = (splitDir.length > 0) ? splitDir.pop() : null // There could be many more directories, but only the top 3 are used for naming /author/series/title/
+  series = splitDir.length > 1 ? splitDir.pop() : null // If there are at least 2 more directories, next furthest will be the series
+  author = splitDir.length > 0 ? splitDir.pop() : null // There could be many more directories, but only the top 3 are used for naming /author/series/title/
 
   // The  may contain various other pieces of metadata, these functions extract it.
   var [folder, asin] = getASIN(folder)
@@ -244,7 +249,6 @@ function getBookDataFromDir(relPath, parseSubtitle = false) {
   var [folder, publishedYear] = getPublishedYear(folder)
   var [title, subtitle] = parseSubtitle ? getSubtitle(folder) : [folder, null]
 
-
   return {
     title,
     subtitle,
@@ -260,8 +264,8 @@ module.exports.getBookDataFromDir = getBookDataFromDir
 
 /**
  * Extract narrator from folder name
- * 
- * @param {string} folder 
+ *
+ * @param {string} folder
  * @returns {[string, string]} [folder, narrator]
  */
 function getNarrator(folder) {
@@ -272,7 +276,7 @@ function getNarrator(folder) {
 
 /**
  * Extract series sequence from folder name
- * 
+ *
  * @example
  * 'Book 2 - Title - Subtitle'
  * 'Title - Subtitle - Vol 12'
@@ -283,8 +287,8 @@ function getNarrator(folder) {
  * '100 - Book Title'
  * '6. Title'
  * '0.5 - Book Title'
- * 
- * @param {string} folder 
+ *
+ * @param {string} folder
  * @returns {[string, string]} [folder, sequence]
  */
 function getSequence(folder) {
@@ -299,7 +303,9 @@ function getSequence(folder) {
     if (match && !(match.groups.suffix && !(match.groups.volumeLabel || match.groups.trailingDot))) {
       volumeNumber = isNaN(match.groups.sequence) ? match.groups.sequence : Number(match.groups.sequence).toString()
       parts[i] = match.groups.suffix
-      if (!parts[i]) { parts.splice(i, 1) }
+      if (!parts[i]) {
+        parts.splice(i, 1)
+      }
       break
     }
   }
@@ -310,8 +316,8 @@ function getSequence(folder) {
 
 /**
  * Extract published year from folder name
- * 
- * @param {string} folder 
+ *
+ * @param {string} folder
  * @returns {[string, string]} [folder, publishedYear]
  */
 function getPublishedYear(folder) {
@@ -329,8 +335,8 @@ function getPublishedYear(folder) {
 
 /**
  * Extract subtitle from folder name
- * 
- * @param {string} folder 
+ *
+ * @param {string} folder
  * @returns {[string, string]} [folder, subtitle]
  */
 function getSubtitle(folder) {
@@ -341,8 +347,8 @@ function getSubtitle(folder) {
 
 /**
  * Extract asin from folder name
- * 
- * @param {string} folder 
+ *
+ * @param {string} folder
  * @returns {[string, string]} [folder, asin]
  */
 function getASIN(folder) {
@@ -358,8 +364,8 @@ function getASIN(folder) {
 }
 
 /**
- * 
- * @param {string} relPath 
+ *
+ * @param {string} relPath
  * @returns {LibraryItemFilenameMetadata}
  */
 function getPodcastDataFromDir(relPath) {
@@ -373,10 +379,10 @@ function getPodcastDataFromDir(relPath) {
 }
 
 /**
- * 
- * @param {string} libraryMediaType 
- * @param {string} folderPath 
- * @param {string} relPath 
+ *
+ * @param {string} libraryMediaType
+ * @param {string} folderPath
+ * @param {string} relPath
  * @returns {{ mediaMetadata: LibraryItemFilenameMetadata, relPath: string, path: string}}
  */
 function getDataFromMediaDir(libraryMediaType, folderPath, relPath) {
@@ -386,7 +392,8 @@ function getDataFromMediaDir(libraryMediaType, folderPath, relPath) {
 
   if (libraryMediaType === 'podcast') {
     mediaMetadata = getPodcastDataFromDir(relPath)
-  } else { // book
+  } else {
+    // book
     mediaMetadata = getBookDataFromDir(relPath, !!global.ServerSettings.scannerParseSubtitle)
   }
 

server/utils/toneHelpers.js

@@ -1,113 +0,0 @@
-const tone = require('node-tone')
-const fs = require('../libs/fsExtra')
-const Logger = require('../Logger')
-
-function getToneMetadataObject(libraryItem, chapters, trackTotal, mimeType = null) {
-  const bookMetadata = libraryItem.media.metadata
-  const coverPath = libraryItem.media.coverPath
-
-  const isMp4 = mimeType === 'audio/mp4'
-  const isMp3 = mimeType === 'audio/mpeg'
-
-  const metadataObject = {
-    'album': bookMetadata.title || '',
-    'title': bookMetadata.title || '',
-    'trackTotal': trackTotal,
-    'additionalFields': {}
-  }
-  if (bookMetadata.subtitle) {
-    metadataObject['subtitle'] = bookMetadata.subtitle
-  }
-  if (bookMetadata.authorName) {
-    metadataObject['artist'] = bookMetadata.authorName
-    metadataObject['albumArtist'] = bookMetadata.authorName
-  }
-  if (bookMetadata.description) {
-    metadataObject['comment'] = bookMetadata.description
-    metadataObject['description'] = bookMetadata.description
-  }
-  if (bookMetadata.narratorName) {
-    metadataObject['narrator'] = bookMetadata.narratorName
-    metadataObject['composer'] = bookMetadata.narratorName
-  }
-  if (bookMetadata.firstSeriesName) {
-    if (!isMp3) {
-      metadataObject.additionalFields['----:com.pilabor.tone:SERIES'] = bookMetadata.firstSeriesName
-    }
-    metadataObject['movementName'] = bookMetadata.firstSeriesName
-  }
-  if (bookMetadata.firstSeriesSequence) {
-    // Non-mp3 
-    if (!isMp3) {
-      metadataObject.additionalFields['----:com.pilabor.tone:PART'] = bookMetadata.firstSeriesSequence
-    }
-    // MP3 Files with non-integer sequence
-    const isNonIntegerSequence = String(bookMetadata.firstSeriesSequence).includes('.') || isNaN(bookMetadata.firstSeriesSequence)
-    if (isMp3 && isNonIntegerSequence) {
-      metadataObject.additionalFields['PART'] = bookMetadata.firstSeriesSequence
-    }
-    if (!isNonIntegerSequence) {
-      metadataObject['movement'] = bookMetadata.firstSeriesSequence
-    }
-  }
-  if (bookMetadata.genres.length) {
-    metadataObject['genre'] = bookMetadata.genres.join('/')
-  }
-  if (bookMetadata.publisher) {
-    metadataObject['publisher'] = bookMetadata.publisher
-  }
-  if (bookMetadata.asin) {
-    if (!isMp3) {
-      metadataObject.additionalFields['----:com.pilabor.tone:AUDIBLE_ASIN'] = bookMetadata.asin
-    }
-    if (!isMp4) {
-      metadataObject.additionalFields['asin'] = bookMetadata.asin
-    }
-  }
-  if (bookMetadata.isbn) {
-    metadataObject.additionalFields['isbn'] = bookMetadata.isbn
-  }
-  if (coverPath) {
-    metadataObject['coverFile'] = coverPath
-  }
-  if (parsePublishedYear(bookMetadata.publishedYear)) {
-    metadataObject['publishingDate'] = parsePublishedYear(bookMetadata.publishedYear)
-  }
-  if (chapters && chapters.length > 0) {
-    let metadataChapters = []
-    for (const chapter of chapters) {
-      metadataChapters.push({
-        start: Math.round(chapter.start * 1000),
-        length: Math.round((chapter.end - chapter.start) * 1000),
-        title: chapter.title,
-      })
-    }
-    metadataObject['chapters'] = metadataChapters
-  }
-
-  return metadataObject
-}
-module.exports.getToneMetadataObject = getToneMetadataObject
-
-module.exports.writeToneMetadataJsonFile = (libraryItem, chapters, filePath, trackTotal, mimeType) => {
-  const metadataObject = getToneMetadataObject(libraryItem, chapters, trackTotal, mimeType)
-  return fs.writeFile(filePath, JSON.stringify({ meta: metadataObject }, null, 2))
-}
-
-module.exports.tagAudioFile = (filePath, payload) => {
-  if (process.env.TONE_PATH) {
-    tone.TONE_PATH = process.env.TONE_PATH
-  }
-
-  return tone.tag(filePath, payload).then((data) => {
-    return true
-  }).catch((error) => {
-    Logger.error(`[toneHelpers] tagAudioFile: Failed for "${filePath}"`, error)
-    return false
-  })
-}
-
-function parsePublishedYear(publishedYear) {
-  if (isNaN(publishedYear) || !publishedYear || Number(publishedYear) <= 0) return null
-  return `01/01/${publishedYear}`
-}

server/utils/toneProber.js

@@ -1,173 +0,0 @@
-const tone = require('node-tone')
-const MediaProbeData = require('../scanner/MediaProbeData')
-const Logger = require('../Logger')
-
-/*
-Sample dump from tone
-{
-  "audio": {
-    "bitrate": 17,
-    "format": "MPEG-4 Part 14",
-    "formatShort": "MPEG-4",
-    "sampleRate": 44100.0,
-    "duration": 209284.0,
-    "channels": {
-      "count": 2,
-      "description": "Stereo (2/0.0)"
-    },
-    "frames": {
-      "offset": 42168,
-      "length": 446932
-    "metaFormat": [
-      "mp4"
-    ]
-  },
-  "meta": {
-    "album": "node-tone",
-    "albumArtist": "advplyr",
-    "artist": "advplyr",
-    "composer": "Composer 5",
-    "comment": "testing out tone metadata",
-    "encodingTool": "audiobookshelf",
-    "genre": "abs",
-    "itunesCompilation": "no",
-    "itunesMediaType": "audiobook",
-    "itunesPlayGap": "noGap",
-    "narrator": "Narrator 5",
-    "recordingDate": "2022-09-10T00:00:00",
-    "title": "Test 5",
-    "trackNumber": 5,
-    "chapters": [
-      {
-        "start": 0,
-        "length": 500,
-        "title": "chapter 1"
-      },
-      {
-        "start": 500,
-        "length": 500,
-        "title": "chapter 2"
-      },
-      {
-        "start": 1000,
-        "length": 208284,
-        "title": "chapter 3"
-      }
-    ],
-    "embeddedPictures": [
-      {
-        "code": 14,
-        "mimetype": "image/png",
-        "data": "..."
-    },
-    "additionalFields": {
-      "test": "Test 5"
-    }
-  },
-  "file": {
-    "size": 530793,
-    "created": "2022-09-10T13:32:51.1942586-05:00",
-    "modified": "2022-09-10T14:09:19.366071-05:00",
-    "accessed": "2022-09-11T13:00:56.5097533-05:00",
-    "path": "C:\\Users\\Coop\\Documents\\NodeProjects\\node-tone\\samples",
-    "name": "m4b.m4b"
-  }
-
-*/
-
-function bitrateKilobitToBit(bitrate) {
-  if (isNaN(bitrate) || !bitrate) return 0
-  return Number(bitrate) * 1000
-}
-
-function msToSeconds(ms) {
-  if (isNaN(ms) || !ms) return 0
-  return Number(ms) / 1000
-}
-
-function parseProbeDump(dumpPayload) {
-  const audioMetadata = dumpPayload.audio
-  const audioChannels = audioMetadata.channels || {}
-  const audio_stream = {
-    bit_rate: bitrateKilobitToBit(audioMetadata.bitrate), // tone uses Kbps but ffprobe uses bps so convert to bits
-    codec: null,
-    time_base: null,
-    language: null,
-    channel_layout: audioChannels.description || null,
-    channels: audioChannels.count || null,
-    sample_rate: audioMetadata.sampleRate || null
-  }
-
-  let chapterIndex = 0
-  const chapters = (dumpPayload.meta.chapters || []).map(chap => {
-    return {
-      id: chapterIndex++,
-      start: msToSeconds(chap.start),
-      end: msToSeconds(chap.start + chap.length),
-      title: chap.title || ''
-    }
-  })
-
-  var video_stream = null
-  if (dumpPayload.meta.embeddedPictures && dumpPayload.meta.embeddedPictures.length) {
-    const mimetype = dumpPayload.meta.embeddedPictures[0].mimetype
-    video_stream = {
-      codec: mimetype === 'image/png' ? 'png' : 'jpeg'
-    }
-  }
-
-  const tags = { ...dumpPayload.meta }
-  delete tags.chapters
-  delete tags.embeddedPictures
-
-  const fileMetadata = dumpPayload.file
-  var sizeBytes = !isNaN(fileMetadata.size) ? Number(fileMetadata.size) : null
-  var sizeMb = sizeBytes !== null ? Number((sizeBytes / (1024 * 1024)).toFixed(2)) : null
-  return {
-    format: audioMetadata.format || 'Unknown',
-    duration: msToSeconds(audioMetadata.duration),
-    size: sizeBytes,
-    sizeMb,
-    bit_rate: audio_stream.bit_rate,
-    audio_stream,
-    video_stream,
-    chapters,
-    tags
-  }
-}
-
-module.exports.probe = (filepath, verbose = false) => {
-  if (process.env.TONE_PATH) {
-    tone.TONE_PATH = process.env.TONE_PATH
-  }
-
-  return tone.dump(filepath).then((dumpPayload) => {
-    if (verbose) {
-      Logger.debug(`[toneProber] dump for file "${filepath}"`, dumpPayload)
-    }
-    const rawProbeData = parseProbeDump(dumpPayload)
-    const probeData = new MediaProbeData()
-    probeData.setDataFromTone(rawProbeData)
-    return probeData
-  }).catch((error) => {
-    Logger.error(`[toneProber] Failed to probe file at path "${filepath}"`, error)
-    return {
-      error
-    }
-  })
-}
-
-module.exports.rawProbe = (filepath) => {
-  if (process.env.TONE_PATH) {
-    tone.TONE_PATH = process.env.TONE_PATH
-  }
-
-  return tone.dump(filepath).then((dumpPayload) => {
-    return dumpPayload
-  }).catch((error) => {
-    Logger.error(`[toneProber] Failed to probe file at path "${filepath}"`, error)
-    return {
-      error
-    }
-  })
-}