Merge branch 'advplyr:master' into master

server/utils/ffmpegHelpers.js

@@ -103,18 +103,39 @@ module.exports.resizeImage = resizeImage
  */
 module.exports.downloadPodcastEpisode = (podcastEpisodeDownload) => {
   return new Promise(async (resolve) => {
-    const response = await axios({
-      url: podcastEpisodeDownload.url,
-      method: 'GET',
-      responseType: 'stream',
-      headers: {
-        'User-Agent': 'audiobookshelf (+https://audiobookshelf.org)'
-      },
-      timeout: global.PodcastDownloadTimeout
-    }).catch((error) => {
-      Logger.error(`[ffmpegHelpers] Failed to download podcast episode with url "${podcastEpisodeDownload.url}"`, error)
-      return null
-    })
+    // Some podcasts fail due to user agent strings
+    // See: https://github.com/advplyr/audiobookshelf/issues/3246 (requires iTMS user agent)
+    // See: https://github.com/advplyr/audiobookshelf/issues/4401 (requires no iTMS user agent)
+    const userAgents = ['audiobookshelf (+https://audiobookshelf.org; like iTMS)', 'audiobookshelf (+https://audiobookshelf.org)']
+
+    let response = null
+    let lastError = null
+
+    for (const userAgent of userAgents) {
+      try {
+        response = await axios({
+          url: podcastEpisodeDownload.url,
+          method: 'GET',
+          responseType: 'stream',
+          headers: {
+            'User-Agent': userAgent
+          },
+          timeout: global.PodcastDownloadTimeout
+        })
+
+        Logger.debug(`[ffmpegHelpers] Successfully connected with User-Agent: ${userAgent}`)
+        break
+      } catch (error) {
+        lastError = error
+        Logger.warn(`[ffmpegHelpers] Failed to download podcast episode with User-Agent "${userAgent}" for url "${podcastEpisodeDownload.url}"`, error.message)
+
+        // If this is the last attempt, log the full error
+        if (userAgent === userAgents[userAgents.length - 1]) {
+          Logger.error(`[ffmpegHelpers] All User-Agent attempts failed for url "${podcastEpisodeDownload.url}"`, lastError)
+        }
+      }
+    }
+
     if (!response) {
       return resolve({
         success: false

server/utils/fileUtils.js

@@ -109,7 +109,7 @@ function getIno(path) {
     .stat(path, { bigint: true })
     .then((data) => String(data.ino))
     .catch((err) => {
-      Logger.error('[Utils] Failed to get ino for path', path, err)
+      Logger.warn(`[Utils] Failed to get ino for path "${path}"`, err)
       return null
     })
 }

server/utils/index.js

@@ -34,6 +34,14 @@ const levenshteinDistance = (str1, str2, caseSensitive = false) => {
 }
 module.exports.levenshteinDistance = levenshteinDistance
 
+const levenshteinSimilarity = (str1, str2, caseSensitive = false) => {
+  const distance = levenshteinDistance(str1, str2, caseSensitive)
+  const maxLength = Math.max(str1.length, str2.length)
+  if (maxLength === 0) return 1
+  return 1 - distance / maxLength
+}
+module.exports.levenshteinSimilarity = levenshteinSimilarity
+
 module.exports.isObject = (val) => {
   return val !== null && typeof val === 'object'
 }

server/utils/notifications.js

@@ -60,6 +60,38 @@ module.exports.notificationData = {
         errorMsg: 'Example error message'
       }
     },
+    {
+      name: 'onRSSFeedFailed',
+      requiresLibrary: true,
+      description: 'Triggered when the RSS feed request fails for an automatic episode download',
+      descriptionKey: 'NotificationOnRSSFeedFailedDescription',
+      variables: ['feedUrl', 'numFailed', 'title'],
+      defaults: {
+        title: 'RSS Feed Request Failed',
+        body: 'Failed to request RSS feed for {{title}}.\nFeed URL: {{feedUrl}}\nNumber of failed attempts: {{numFailed}}'
+      },
+      testData: {
+        title: 'Test RSS Feed',
+        feedUrl: 'https://example.com/rss',
+        numFailed: 3
+      }
+    },
+    {
+      name: 'onRSSFeedDisabled',
+      requiresLibrary: true,
+      description: 'Triggered when automatic episode downloads are disabled due to too many failed attempts',
+      descriptionKey: 'NotificationOnRSSFeedDisabledDescription',
+      variables: ['feedUrl', 'numFailed', 'title'],
+      defaults: {
+        title: 'Podcast Episode Download Schedule Disabled',
+        body: 'Automatic episode downloads for {{title}} have been disabled due to too many failed RSS feed requests.\nFeed URL: {{feedUrl}}\nNumber of failed attempts: {{numFailed}}'
+      },
+      testData: {
+        title: 'Test RSS Feed',
+        feedUrl: 'https://example.com/rss',
+        numFailed: 5
+      }
+    },
     {
       name: 'onTest',
       requiresLibrary: false,

server/utils/podcastUtils.js

@@ -1,8 +1,9 @@
 const axios = require('axios')
 const ssrfFilter = require('ssrf-req-filter')
 const Logger = require('../Logger')
-const { xmlToJSON, levenshteinDistance, timestampToSeconds } = require('./index')
+const { xmlToJSON, timestampToSeconds } = require('./index')
 const htmlSanitizer = require('../utils/htmlSanitizer')
+const Fuse = require('../libs/fusejs')
 
 /**
  * @typedef RssPodcastChapter
@@ -24,6 +25,7 @@ const htmlSanitizer = require('../utils/htmlSanitizer')
  * @property {string} episode
  * @property {string} author
  * @property {string} duration
+ * @property {number|null} durationSeconds - Parsed from duration string if duration is valid
  * @property {string} explicit
  * @property {number} publishedAt - Unix timestamp
  * @property {{ url: string, type?: string, length?: string }} enclosure
@@ -205,7 +207,7 @@ function extractEpisodeData(item) {
     } else if (typeof guidItem?._ === 'string') {
       episode.guid = guidItem._
     } else {
-      Logger.error(`[podcastUtils] Invalid guid ${item['guid']} for ${episode.enclosure.url}`)
+      Logger.error(`[podcastUtils] Invalid guid for ${episode.enclosure.url}`, item['guid'])
     }
   }
 
@@ -216,8 +218,9 @@ function extractEpisodeData(item) {
   })
 
   // Extract psc:chapters if duration is set
-  let episodeDuration = !isNaN(episode.duration) ? timestampToSeconds(episode.duration) : null
-  if (item['psc:chapters']?.[0]?.['psc:chapter']?.length && episodeDuration) {
+  episode.durationSeconds = episode.duration ? timestampToSeconds(episode.duration) : null
+
+  if (item['psc:chapters']?.[0]?.['psc:chapter']?.length && episode.durationSeconds) {
     // Example chapter:
     // {"id":0,"start":0,"end":43.004286,"title":"chapter 1"}
 
@@ -243,7 +246,7 @@ function extractEpisodeData(item) {
     } else {
       episode.chapters = cleanedChapters.map((chapter, index) => {
         const nextChapter = cleanedChapters[index + 1]
-        const end = nextChapter ? nextChapter.start : episodeDuration
+        const end = nextChapter ? nextChapter.start : episode.durationSeconds
         return {
           id: chapter.id,
           title: chapter.title,
@@ -272,6 +275,7 @@ function cleanEpisodeData(data) {
     episode: data.episode || '',
     author: data.author || '',
     duration: data.duration || '',
+    durationSeconds: data.durationSeconds || null,
     explicit: data.explicit || '',
     publishedAt,
     enclosure: data.enclosure,
@@ -407,7 +411,7 @@ module.exports.getPodcastFeed = (feedUrl, excludeEpisodeMetadata = false) => {
     })
 }
 
-// Return array of episodes ordered by closest match (Levenshtein distance of 6 or less)
+// Return array of episodes ordered by closest match using fuse.js
 module.exports.findMatchingEpisodes = async (feedUrl, searchTitle) => {
   const feed = await this.getPodcastFeed(feedUrl).catch(() => {
     return null
@@ -420,32 +424,29 @@ module.exports.findMatchingEpisodes = async (feedUrl, searchTitle) => {
  *
  * @param {RssPodcast} feed
  * @param {string} searchTitle
- * @returns {Array<{ episode: RssPodcastEpisode, levenshtein: number }>}
+ * @param {number} [threshold=0.4] - 0.0 for perfect match, 1.0 for match anything
+ * @returns {Array<{ episode: RssPodcastEpisode }>}
  */
-module.exports.findMatchingEpisodesInFeed = (feed, searchTitle) => {
-  searchTitle = searchTitle.toLowerCase().trim()
+module.exports.findMatchingEpisodesInFeed = (feed, searchTitle, threshold = 0.4) => {
   if (!feed?.episodes) {
     return null
   }
 
+  const fuseOptions = {
+    ignoreDiacritics: true,
+    threshold,
+    keys: [
+      { name: 'title', weight: 0.7 }, // prefer match in title
+      { name: 'subtitle', weight: 0.3 }
+    ]
+  }
+  const fuse = new Fuse(feed.episodes, fuseOptions)
+
   const matches = []
-  feed.episodes.forEach((ep) => {
-    if (!ep.title) return
-    const epTitle = ep.title.toLowerCase().trim()
-    if (epTitle === searchTitle) {
-      matches.push({
-        episode: ep,
-        levenshtein: 0
-      })
-    } else {
-      const levenshtein = levenshteinDistance(searchTitle, epTitle, true)
-      if (levenshtein <= 6 && epTitle.length > levenshtein) {
-        matches.push({
-          episode: ep,
-          levenshtein
-        })
-      }
-    }
+  fuse.search(searchTitle).forEach((match) => {
+    matches.push({
+      episode: match.item
+    })
   })
-  return matches.sort((a, b) => a.levenshtein - b.levenshtein)
+  return matches
 }

server/utils/queries/libraryItemsBookFilters.js

@@ -186,6 +186,8 @@ module.exports = {
       mediaWhere['$series.id$'] = null
     } else if (group === 'abridged') {
       mediaWhere['abridged'] = true
+    } else if (group === 'explicit') {
+      mediaWhere['explicit'] = true
     } else if (['genres', 'tags', 'narrators'].includes(group)) {
       mediaWhere[group] = Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM json_each(${group}) WHERE json_valid(${group}) AND json_each.value = :filterValue)`), {
         [Sequelize.Op.gte]: 1
@@ -251,6 +253,15 @@ module.exports = {
    */
   getOrder(sortBy, sortDesc, collapseseries) {
     const dir = sortDesc ? 'DESC' : 'ASC'
+
+    const getTitleOrder = () => {
+      if (global.ServerSettings.sortingIgnorePrefix) {
+        return [Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]
+      } else {
+        return [Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]
+      }
+    }
+
     if (sortBy === 'addedAt') {
       return [[Sequelize.literal('libraryItem.createdAt'), dir]]
     } else if (sortBy === 'size') {
@@ -264,19 +275,16 @@ module.exports = {
     } else if (sortBy === 'media.metadata.publishedYear') {
       return [[Sequelize.literal(`CAST(\`book\`.\`publishedYear\` AS INTEGER)`), dir]]
     } else if (sortBy === 'media.metadata.authorNameLF') {
-      return [[Sequelize.literal('`libraryItem`.`authorNamesLastFirst` COLLATE NOCASE'), dir]]
+      // Sort by author name last first, secondary sort by title
+      return [[Sequelize.literal('`libraryItem`.`authorNamesLastFirst` COLLATE NOCASE'), dir], getTitleOrder()]
     } else if (sortBy === 'media.metadata.authorName') {
-      return [[Sequelize.literal('`libraryItem`.`authorNamesFirstLast` COLLATE NOCASE'), dir]]
+      // Sort by author name first last, secondary sort by title
+      return [[Sequelize.literal('`libraryItem`.`authorNamesFirstLast` COLLATE NOCASE'), dir], getTitleOrder()]
     } else if (sortBy === 'media.metadata.title') {
       if (collapseseries) {
         return [[Sequelize.literal('display_title COLLATE NOCASE'), dir]]
       }
-
-      if (global.ServerSettings.sortingIgnorePrefix) {
-        return [[Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]]
-      } else {
-        return [[Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]]
-      }
+      return [getTitleOrder()]
     } else if (sortBy === 'sequence') {
       const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'
       return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]]

server/utils/queries/libraryItemsPodcastFilters.js

@@ -59,6 +59,8 @@ module.exports = {
       replacements.filterValue = value
     } else if (group === 'languages') {
       mediaWhere['language'] = value
+    } else if (group === 'explicit') {
+      mediaWhere['explicit'] = true
     }
 
     return {
@@ -149,11 +151,12 @@ module.exports = {
       libraryId
     }
     const libraryItemIncludes = []
-    if (includeRSSFeed) {
+    if (filterGroup === 'feed-open' || includeRSSFeed) {
+      const rssFeedRequired = filterGroup === 'feed-open'
       libraryItemIncludes.push({
         model: Database.feedModel,
-        required: filterGroup === 'feed-open',
-        separate: true
+        required: rssFeedRequired,
+        separate: !rssFeedRequired
       })
     }
     if (filterGroup === 'issues') {

server/utils/rateLimiterFactory.js

@@ -0,0 +1,82 @@
+const { rateLimit, RateLimitRequestHandler } = require('express-rate-limit')
+const Logger = require('../Logger')
+const requestIp = require('../libs/requestIp')
+
+/**
+ * Factory for creating authentication rate limiters
+ */
+class RateLimiterFactory {
+  static DEFAULT_WINDOW_MS = 10 * 60 * 1000 // 10 minutes
+  static DEFAULT_MAX = 40 // 40 attempts
+
+  constructor() {
+    this.authRateLimiter = null
+  }
+
+  /**
+   * Get the authentication rate limiter
+   * @returns {RateLimitRequestHandler}
+   */
+  getAuthRateLimiter() {
+    if (this.authRateLimiter) {
+      return this.authRateLimiter
+    }
+
+    // Disable by setting max to 0
+    if (process.env.RATE_LIMIT_AUTH_MAX === '0') {
+      this.authRateLimiter = (req, res, next) => next()
+      Logger.info(`[RateLimiterFactory] Authentication rate limiting disabled by ENV variable`)
+      return this.authRateLimiter
+    }
+
+    let windowMs = RateLimiterFactory.DEFAULT_WINDOW_MS
+    if (parseInt(process.env.RATE_LIMIT_AUTH_WINDOW) > 0) {
+      windowMs = parseInt(process.env.RATE_LIMIT_AUTH_WINDOW)
+      if (windowMs !== RateLimiterFactory.DEFAULT_WINDOW_MS) {
+        Logger.info(`[RateLimiterFactory] Authentication rate limiting window set to ${windowMs}ms by ENV variable`)
+      }
+    }
+
+    let max = RateLimiterFactory.DEFAULT_MAX
+    if (parseInt(process.env.RATE_LIMIT_AUTH_MAX) > 0) {
+      max = parseInt(process.env.RATE_LIMIT_AUTH_MAX)
+      if (max !== RateLimiterFactory.DEFAULT_MAX) {
+        Logger.info(`[RateLimiterFactory] Authentication rate limiting max set to ${max} by ENV variable`)
+      }
+    }
+
+    let message = 'Too many authentication requests'
+    if (process.env.RATE_LIMIT_AUTH_MESSAGE) {
+      message = process.env.RATE_LIMIT_AUTH_MESSAGE
+    }
+
+    this.authRateLimiter = rateLimit({
+      windowMs,
+      max,
+      standardHeaders: true,
+      legacyHeaders: false,
+      keyGenerator: (req) => {
+        // Override keyGenerator to handle proxy IPs
+        return requestIp.getClientIp(req) || req.ip
+      },
+      handler: (req, res) => {
+        const userAgent = req.get('User-Agent') || 'Unknown'
+        const endpoint = req.path
+        const method = req.method
+        const ip = requestIp.getClientIp(req) || req.ip
+
+        Logger.warn(`[RateLimiter] Rate limit exceeded - IP: ${ip}, Endpoint: ${method} ${endpoint}, User-Agent: ${userAgent}`)
+
+        res.status(429).json({
+          error: message
+        })
+      }
+    })
+
+    Logger.debug(`[RateLimiterFactory] Created auth rate limiter: ${max} attempts per ${windowMs / 1000 / 60} minutes`)
+
+    return this.authRateLimiter
+  }
+}
+
+module.exports = new RateLimiterFactory()