Merge branch 'advplyr:master' into master

server/controllers/MiscController.js

@@ -19,6 +19,10 @@ class MiscController {
       Logger.warn('User attempted to upload without permission', req.user)
       return res.sendStatus(403)
     }
+    if (!req.files) {
+      Logger.error('Invalid request, no files')
+      return res.sendStatus(400)
+    }
     var files = Object.values(req.files)
     var title = req.body.title
     var author = req.body.author

server/controllers/PodcastController.js

@@ -31,21 +31,24 @@ class PodcastController {
     }
 
     const podcastPath = filePathToPOSIX(payload.path)
-    if (await fs.pathExists(podcastPath)) {
-      Logger.error(`[PodcastController] Podcast folder already exists "${podcastPath}"`)
+
+    // Check if a library item with this podcast folder exists already
+    const existingLibraryItem = this.db.libraryItems.find(li => li.path === podcastPath && li.libraryId === library.id)
+    if (existingLibraryItem) {
+      Logger.error(`[PodcastController] Podcast already exists with name "${existingLibraryItem.media.metadata.title}" at path "${podcastPath}"`)
       return res.status(400).send('Podcast already exists')
     }
 
-    var success = await fs.ensureDir(podcastPath).then(() => true).catch((error) => {
+    const success = await fs.ensureDir(podcastPath).then(() => true).catch((error) => {
       Logger.error(`[PodcastController] Failed to ensure podcast dir "${podcastPath}"`, error)
       return false
     })
     if (!success) return res.status(400).send('Invalid podcast path')
     await filePerms.setDefault(podcastPath)
 
-    var libraryItemFolderStats = await getFileTimestampsWithIno(podcastPath)
+    const libraryItemFolderStats = await getFileTimestampsWithIno(podcastPath)
 
-    var relPath = payload.path.replace(folder.fullPath, '')
+    let relPath = payload.path.replace(folder.fullPath, '')
     if (relPath.startsWith('/')) relPath = relPath.slice(1)
 
     const libraryItemPayload = {
@@ -60,14 +63,14 @@ class PodcastController {
       media: payload.media
     }
 
-    var libraryItem = new LibraryItem()
+    const libraryItem = new LibraryItem()
     libraryItem.setData('podcast', libraryItemPayload)
 
     // Download and save cover image
     if (payload.media.metadata.imageUrl) {
       // TODO: Scan cover image to library files
       // Podcast cover will always go into library item folder
-      var coverResponse = await this.coverManager.downloadCoverFromUrl(libraryItem, payload.media.metadata.imageUrl, true)
+      const coverResponse = await this.coverManager.downloadCoverFromUrl(libraryItem, payload.media.metadata.imageUrl, true)
       if (coverResponse) {
         if (coverResponse.error) {
           Logger.error(`[PodcastController] Download cover error from "${payload.media.metadata.imageUrl}": ${coverResponse.error}`)

server/finders/BookFinder.js

@@ -3,6 +3,7 @@ const GoogleBooks = require('../providers/GoogleBooks')
 const Audible = require('../providers/Audible')
 const iTunes = require('../providers/iTunes')
 const Audnexus = require('../providers/Audnexus')
+const FantLab = require('../providers/FantLab')
 const Logger = require('../Logger')
 const { levenshteinDistance } = require('../utils/index')
 
@@ -13,6 +14,7 @@ class BookFinder {
     this.audible = new Audible()
     this.iTunesApi = new iTunes()
     this.audnexus = new Audnexus()
+    this.fantLab = new FantLab()
 
     this.verbose = false
   }
@@ -146,6 +148,17 @@ class BookFinder {
     return books
   }
 
+  async getFantLabResults(title, author) {
+    var books = await this.fantLab.search(title, author)
+    if (this.verbose) Logger.debug(`FantLab Book Search Results: ${books.length || 0}`)
+    if (books.errorCode) {
+      Logger.error(`FantLab Search Error ${books.errorCode}`)
+      return []
+    }
+
+    return books
+  }
+
   async getiTunesAudiobooksResults(title, author) {
     return this.iTunesApi.searchAudiobooks(title)
   }
@@ -172,7 +185,10 @@ class BookFinder {
       books = await this.getiTunesAudiobooksResults(title, author)
     } else if (provider === 'openlibrary') {
       books = await this.getOpenLibResults(title, author, maxTitleDistance, maxAuthorDistance)
-    } else {
+    } else if (provider === 'fantlab') {
+      books = await this.getFantLabResults(title, author)
+    }
+    else {
       books = await this.getGoogleBooksResults(title, author)
     }
 
@@ -186,7 +202,7 @@ class BookFinder {
       return this.search(provider, cleanedTitle, cleanedAuthor, isbn, asin, options)
     }
 
-    if (["google", "audible", "itunes"].includes(provider)) return books
+    if (["google", "audible", "itunes", 'fantlab'].includes(provider)) return books
 
     return books.sort((a, b) => {
       return a.totalDistance - b.totalDistance

server/managers/BackupManager.js

@@ -295,14 +295,16 @@ class BackupManager {
       // pipe archive data to the file
       archive.pipe(output)
 
-      archive.directory(this.db.LibraryItemsPath, 'config/libraryItems/data')
-      archive.directory(this.db.UsersPath, 'config/users/data')
-      archive.directory(this.db.SessionsPath, 'config/sessions/data')
-      archive.directory(this.db.LibrariesPath, 'config/libraries/data')
-      archive.directory(this.db.SettingsPath, 'config/settings/data')
-      archive.directory(this.db.CollectionsPath, 'config/collections/data')
-      archive.directory(this.db.AuthorsPath, 'config/authors/data')
-      archive.directory(this.db.SeriesPath, 'config/series/data')
+      archive.directory(Path.join(this.db.LibraryItemsPath, 'data'), 'config/libraryItems/data')
+      archive.directory(Path.join(this.db.UsersPath, 'data'), 'config/users/data')
+      archive.directory(Path.join(this.db.SessionsPath, 'data'), 'config/sessions/data')
+      archive.directory(Path.join(this.db.LibrariesPath, 'data'), 'config/libraries/data')
+      archive.directory(Path.join(this.db.SettingsPath, 'data'), 'config/settings/data')
+      archive.directory(Path.join(this.db.CollectionsPath, 'data'), 'config/collections/data')
+      archive.directory(Path.join(this.db.AuthorsPath, 'data'), 'config/authors/data')
+      archive.directory(Path.join(this.db.SeriesPath, 'data'), 'config/series/data')
+      archive.directory(Path.join(this.db.PlaylistsPath, 'data'), 'config/playlists/data')
+      archive.directory(Path.join(this.db.FeedsPath, 'data'), 'config/feeds/data')
 
       if (this.serverSettings.backupMetadataCovers) {
         Logger.debug(`[BackupManager] Backing up Metadata Items "${this.ItemsMetadataPath}"`)

server/managers/CoverManager.js

@@ -54,7 +54,7 @@ class CoverManager {
     for (let i = 0; i < filesInDir.length; i++) {
       var file = filesInDir[i]
       var _extname = Path.extname(file).toLowerCase()
-      var _filename = Path.basename(file, _extname)
+      var _filename = Path.basename(file, _extname).toLowerCase()
       if (_filename === 'cover' && _extname !== newCoverExt && imageExtensions.includes(_extname)) {
         var filepath = Path.join(dirpath, file)
         Logger.debug(`[CoverManager] Removing old cover from metadata "${filepath}"`)
@@ -83,20 +83,20 @@ class CoverManager {
   }
 
   async uploadCover(libraryItem, coverFile) {
-    var extname = Path.extname(coverFile.name.toLowerCase())
+    const extname = Path.extname(coverFile.name.toLowerCase())
     if (!extname || !globals.SupportedImageTypes.includes(extname.slice(1))) {
       return {
         error: `Invalid image type ${extname} (Supported: ${globals.SupportedImageTypes.join(',')})`
       }
     }
 
-    var coverDirPath = this.getCoverDirectory(libraryItem)
+    const coverDirPath = this.getCoverDirectory(libraryItem)
     await fs.ensureDir(coverDirPath)
 
-    var coverFullPath = Path.posix.join(coverDirPath, `cover${extname}`)
+    const coverFullPath = Path.posix.join(coverDirPath, `cover${extname}`)
 
     // Move cover from temp upload dir to destination
-    var success = await coverFile.mv(coverFullPath).then(() => true).catch((error) => {
+    const success = await coverFile.mv(coverFullPath).then(() => true).catch((error) => {
       Logger.error('[CoverManager] Failed to move cover file', path, error)
       return false
     })

server/managers/NotificationManager.js

@@ -24,9 +24,15 @@ class NotificationManager {
       libraryItemId: libraryItem.id,
       libraryId: libraryItem.libraryId,
       libraryName: library ? library.name : 'Unknown',
+      mediaTags: (libraryItem.media.tags || []).join(', '),
       podcastTitle: libraryItem.media.metadata.title,
+      podcastAuthor: libraryItem.media.metadata.author || '',
+      podcastDescription: libraryItem.media.metadata.description || '',
+      podcastGenres: (libraryItem.media.metadata.genres || []).join(', '),
       episodeId: episode.id,
-      episodeTitle: episode.title
+      episodeTitle: episode.title,
+      episodeSubtitle: episode.subtitle || '',
+      episodeDescription: episode.description || ''
     }
     this.triggerNotification('onPodcastEpisodeDownloaded', eventData)
   }
@@ -110,4 +116,4 @@ class NotificationManager {
     })
   }
 }
-module.exports = NotificationManager
+module.exports = NotificationManager

server/providers/FantLab.js

@@ -0,0 +1,152 @@
+const axios = require('axios')
+const Logger = require('../Logger')
+
+class FantLab {
+  // 7 - other
+  // 11 - essay
+  // 12 - article
+  // 22 - disser
+  // 23 - monography
+  // 24 - study
+  // 25 - encyclopedy
+  // 26 - magazine
+  // 46 - sketch
+  // 47 - reportage
+  // 49 - excerpt
+  // 51 - interview
+  // 52 - review
+  // 55 - libretto
+  // 56 - anthology series
+  // 57 - newspaper
+  // types can get here https://api.fantlab.ru/config.json
+  _filterWorkType = [7, 11, 12, 22, 23, 24, 25, 26, 46, 47, 49, 51, 52, 55, 56, 57]
+  _baseUrl = 'https://api.fantlab.ru'
+
+  constructor() { }
+
+  async search(title, author) {
+    let searchString = encodeURIComponent(title)
+    if (author) {
+      searchString += encodeURIComponent(' ' + author)
+    }
+    const url = `${this._baseUrl}/search-works?q=${searchString}&page=1&onlymatches=1`
+    Logger.debug(`[FantLab] Search url: ${url}`)
+    const items = await axios.get(url).then((res) => {
+      return res.data || []
+    }).catch(error => {
+      Logger.error('[FantLab] search error', error)
+      return []
+    })
+
+    return Promise.all(items.map(async item => await this.getWork(item))).then(resArray => {
+      return resArray.filter(res => res)
+    })
+  }
+
+  async getWork(item) {
+    const { work_id, work_type_id } = item
+
+    if (this._filterWorkType.includes(work_type_id)) {
+      return null
+    }
+
+    const url = `${this._baseUrl}/work/${work_id}/extended`
+    const bookData = await axios.get(url).then((resp) => {
+      return resp.data || null
+    }).catch((error) => {
+      Logger.error(`[FantLab] work info request for url "${url}" error`, error)
+      return null
+    })
+
+    return this.cleanBookData(bookData)
+  }
+
+  async cleanBookData(bookData) {
+    let { authors, work_name_alts, work_id, work_name, work_year, work_description, image, classificatory, editions_blocks } = bookData
+
+    const subtitle = Array.isArray(work_name_alts) ? work_name_alts[0] : null
+    const authorNames = authors.map(au => (au.name || '').trim()).filter(au => au)
+
+    const imageAndIsbn = await this.tryGetCoverFromEditions(editions_blocks)
+
+    const imageToUse = imageAndIsbn?.imageUrl || image
+
+    return {
+      id: work_id,
+      title: work_name,
+      subtitle: subtitle || null,
+      author: authorNames.length ? authorNames.join(', ') : null,
+      publisher: null,
+      publishedYear: work_year,
+      description: work_description,
+      cover: imageToUse ? `https://fantlab.ru${imageToUse}` : null,
+      genres: this.tryGetGenres(classificatory),
+      isbn: imageAndIsbn?.isbn || null
+    }
+  }
+
+  tryGetGenres(classificatory) {
+    if (!classificatory || !classificatory.genre_group) return []
+
+    const genresGroup = classificatory.genre_group.find(group => group.genre_group_id == 1) // genres and subgenres
+
+    // genre_group_id=2 - General Characteristics
+    // genre_group_id=3 - Arena
+    // genre_group_id=4 - Duration of action
+    // genre_group_id=6 - Story moves
+    // genre_group_id=7 - Story linearity
+    // genre_group_id=5 - Recommended age of the reader
+
+    if (!genresGroup || !genresGroup.genre || !genresGroup.genre.length) return []
+
+    const rootGenre = genresGroup.genre[0]
+
+    const { label } = rootGenre
+
+    return [label].concat(this.tryGetSubGenres(rootGenre))
+  }
+
+  tryGetSubGenres(rootGenre) {
+    if (!rootGenre.genre || !rootGenre.genre.length) return []
+    return rootGenre.genre.map(g => g.label).filter(g => g)
+  }
+
+  async tryGetCoverFromEditions(editions) {
+    if (!editions) {
+      return null
+    }
+
+    // 30 = audio, 10 = paper
+    // Prefer audio if available
+    const bookEditions = editions['30'] || editions['10']
+    if (!bookEditions || !bookEditions.list || !bookEditions.list.length) {
+      return null
+    }
+
+    const lastEdition = bookEditions.list.pop()
+
+    const editionId = lastEdition['edition_id']
+    const isbn = lastEdition['isbn'] || null // get only from paper edition
+
+    return {
+      imageUrl: await this.getCoverFromEdition(editionId),
+      isbn
+    }
+  }
+
+  async getCoverFromEdition(editionId) {
+    if (!editionId) return null
+    const url = `${this._baseUrl}/edition/${editionId}`
+
+    const editionInfo = await axios.get(url).then((resp) => {
+      return resp.data || null
+    }).catch(error => {
+      Logger.error(`[FantLab] search cover from edition with url "${url}" error`, error)
+      return null
+    })
+
+    return editionInfo?.image || null
+  }
+}
+
+module.exports = FantLab

server/scanner/MediaFileScanner.js

@@ -115,11 +115,16 @@ class MediaFileScanner {
 
     const scanStart = Date.now()
     const mediaMetadata = libraryItem.media.metadata || null
-    const proms = []
-    for (let i = 0; i < mediaLibraryFiles.length; i++) {
-      proms.push(this.scan(mediaType, mediaLibraryFiles[i], mediaMetadata))
+    const batchSize = 32
+    const results = []
+    for (let batch = 0; batch < mediaLibraryFiles.length; batch += batchSize) {
+      const proms = []
+      for (let i = batch; i < Math.min(batch + batchSize, mediaLibraryFiles.length); i++) {
+        proms.push(this.scan(mediaType, mediaLibraryFiles[i], mediaMetadata))
+      }
+      results.push(...await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr)))
     }
-    const results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr))
+
     return {
       audioFiles: results.filter(r => r.audioFile).map(r => r.audioFile),
       videoFiles: results.filter(r => r.videoFile).map(r => r.videoFile),

server/scanner/Scanner.js

@@ -581,7 +581,7 @@ class Scanner {
       if (!existingLibraryItem) {
         existingLibraryItem = this.db.libraryItems.find(li => li.ino === dirIno)
         if (existingLibraryItem) {
-          Logger.debug(`[Scanner] scanFolderUpdates: Library item found by inode value "${existingLibraryItem.relPath} => ${itemDir}"`)
+          Logger.debug(`[Scanner] scanFolderUpdates: Library item found by inode value=${dirIno}. "${existingLibraryItem.relPath} => ${itemDir}"`)
           // Update library item paths for scan and all library item paths will get updated in LibraryItem.checkScanData
           existingLibraryItem.path = fullPath
           existingLibraryItem.relPath = itemDir

server/utils/notifications.js

@@ -7,7 +7,7 @@ module.exports.notificationData = {
       requiresLibrary: true,
       libraryMediaType: 'podcast',
       description: 'Triggered when a podcast episode is auto-downloaded',
-      variables: ['libraryItemId', 'libraryId', 'podcastTitle', 'episodeTitle', 'libraryName', 'episodeId'],
+      variables: ['libraryItemId', 'libraryId', 'podcastTitle', 'podcastAuthor', 'podcastDescription', 'podcastGenres', 'episodeTitle', 'episodeSubtitle', 'episodeDescription', 'libraryName', 'episodeId', 'mediaTags'],
       defaults: {
         title: 'New {{podcastTitle}} Episode!',
         body: '{{episodeTitle}} has been added to {{libraryName}} library.'
@@ -16,9 +16,15 @@ module.exports.notificationData = {
         libraryItemId: 'li_notification_test',
         libraryId: 'lib_test',
         libraryName: 'Podcasts',
+        mediaTags: 'TestTag1, TestTag2',
         podcastTitle: 'Abs Test Podcast',
+        podcastAuthor: 'Audiobookshelf',
+        podcastDescription: 'Description of the Abs Test Podcast belongs here.',
+        podcastGenres: 'TestGenre1, TestGenre2',
         episodeId: 'ep_notification_test',
-        episodeTitle: 'Successful Test'
+        episodeTitle: 'Successful Test Episode',
+        episodeSubtitle: 'Episode Subtitle',
+        episodeDescription: 'Some description of the podcast episode.'
       }
     },
     {
@@ -35,4 +41,4 @@ module.exports.notificationData = {
       }
     }
   ]
-}
+}

server/utils/parsers/parseOpfMetadata.js

@@ -22,7 +22,27 @@ function fetchCreators(creators, role) {
 
 function fetchTagString(metadata, tag) {
   if (!metadata[tag] || !metadata[tag].length) return null
-  const value = metadata[tag][0]
+  let value = metadata[tag][0]
+
+  /*
+    EXAMPLES:
+
+    "dc:title": [
+      {
+        "_": "The Quest for Character",
+        "$": {
+          "opf:file-as": "Quest for Character What the Story of Socrates and Alcibiades"
+        }
+      }
+    ]
+
+    OR
+
+    "dc:title": [
+      "The Quest for Character"
+    ]
+  */
+  if (typeof value === 'object') value = value._
   if (typeof value !== 'string') return null
   return value
 }

server/utils/podcastUtils.js

@@ -191,7 +191,17 @@ module.exports.parsePodcastRssFeedXml = async (xml, excludeEpisodeMetadata = fal
 
 module.exports.getPodcastFeed = (feedUrl, excludeEpisodeMetadata = false) => {
   Logger.debug(`[podcastUtils] getPodcastFeed for "${feedUrl}"`)
-  return axios.get(feedUrl, { timeout: 6000 }).then(async (data) => {
+  return axios.get(feedUrl, { timeout: 6000, responseType: 'arraybuffer' }).then(async (data) => {
+
+    // Adding support for ios-8859-1 encoded RSS feeds.
+    //  See: https://github.com/advplyr/audiobookshelf/issues/1489
+    const contentType = data.headers?.['content-type'] || '' // e.g. text/xml; charset=iso-8859-1
+    if (contentType.toLowerCase().includes('iso-8859-1')) {
+      data.data = data.data.toString('latin1')
+    } else {
+      data.data = data.data.toString()
+    }
+
     if (!data || !data.data) {
       Logger.error(`[podcastUtils] getPodcastFeed: Invalid podcast feed request response (${feedUrl})`)
       return false