Merge c0319ebbac into 1d0b7e383a

2026-02-28 21:19:42 +00:00 · 2026-02-24 00:22:55 +00:00 · 2026-02-24 00:22:55 +00:00 · fd98d17d0c
commit fd98d17d0c
parent 1d0b7e383a c0319ebbac
4 changed files with 320 additions and 105 deletions
--- a/server/managers/ApiCacheManager.js
+++ b/server/managers/ApiCacheManager.js
@ -5,6 +5,9 @@ const Database = require('../Database')
 class ApiCacheManager {
  defaultCacheOptions = { max: 1000, maxSize: 10 * 1000 * 1000, sizeCalculation: (item) => item.body.length + JSON.stringify(item.headers).length }
  defaultTtlOptions = { ttl: 30 * 60 * 1000 }
+  highChurnModels = new Set(['session', 'mediaProgress', 'playbackSession', 'device'])
+  modelsInvalidatingPersonalized = new Set(['mediaProgress'])
+  modelsInvalidatingMe = new Set(['session', 'mediaProgress', 'playbackSession', 'device'])

  constructor(cache = new LRUCache(this.defaultCacheOptions), ttlOptions = this.defaultTtlOptions) {
    this.cache = cache
@ -16,8 +19,44 @@ class ApiCacheManager {
    hooks.forEach((hook) => database.sequelize.addHook(hook, (model) => this.clear(model, hook)))
  }

+  getModelName(model) {
+    if (typeof model?.name === 'string') return model.name
+    if (typeof model?.model?.name === 'string') return model.model.name
+    if (typeof model?.constructor?.name === 'string' && model.constructor.name !== 'Object') return model.constructor.name
+    return 'unknown'
+  }
+
+  clearByUrlPattern(urlPattern) {
+    let removed = 0
+    for (const key of this.cache.keys()) {
+      try {
+        const parsed = JSON.parse(key)
+        if (typeof parsed?.url === 'string' && urlPattern.test(parsed.url)) {
+          if (this.cache.delete(key)) removed++
+        }
+      } catch {
+        if (this.cache.delete(key)) removed++
+      }
+    }
+    return removed
+  }
+
+  clearUserProgressSlices(modelName, hook) {
+    const removedPersonalized = this.modelsInvalidatingPersonalized.has(modelName) ? this.clearByUrlPattern(/^\/libraries\/[^/]+\/personalized/) : 0
+    const removedMe = this.modelsInvalidatingMe.has(modelName) ? this.clearByUrlPattern(/^\/me(\/|\?|$)/) : 0
+    Logger.debug(
+      `[ApiCacheManager] ${modelName}.${hook}: cleared user-progress cache slices (personalized=${removedPersonalized}, me=${removedMe})`
+    )
+  }
+
  clear(model, hook) {
-    Logger.debug(`[ApiCacheManager] ${model.constructor.name}.${hook}: Clearing cache`)
+    const modelName = this.getModelName(model)
+    if (this.highChurnModels.has(modelName)) {
+      this.clearUserProgressSlices(modelName, hook)
+      return
+    }
+
+    Logger.debug(`[ApiCacheManager] ${modelName}.${hook}: Clearing cache`)
    this.cache.clear()
  }

--- a/server/migrations/v2.32.2-add-discover-query-indexes.js
+++ b/server/migrations/v2.32.2-add-discover-query-indexes.js
@ -0,0 +1,74 @@
+/**
+ * @typedef MigrationContext
+ * @property {import('sequelize').QueryInterface} queryInterface
+ * @property {import('../Logger')} logger
+ *
+ * @typedef MigrationOptions
+ * @property {MigrationContext} context
+ */
+
+const migrationVersion = '2.32.2'
+const migrationName = `${migrationVersion}-add-discover-query-indexes`
+const loggerPrefix = `[${migrationVersion} migration]`
+
+const indexes = [
+  {
+    table: 'mediaProgresses',
+    name: 'media_progress_user_item_finished_time',
+    fields: ['userId', 'mediaItemId', 'isFinished', 'currentTime']
+  },
+  {
+    table: 'bookSeries',
+    name: 'book_series_series_book',
+    fields: ['seriesId', 'bookId']
+  }
+]
+
+async function up({ context: { queryInterface, logger } }) {
+  logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
+
+  for (const index of indexes) {
+    await addIndexIfMissing(queryInterface, logger, index)
+  }
+
+  logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
+}
+
+async function down({ context: { queryInterface, logger } }) {
+  logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
+
+  for (const index of indexes) {
+    await removeIndexIfExists(queryInterface, logger, index)
+  }
+
+  logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
+}
+
+async function addIndexIfMissing(queryInterface, logger, index) {
+  const existing = await queryInterface.showIndex(index.table)
+  if (existing.some((i) => i.name === index.name)) {
+    logger.info(`${loggerPrefix} index ${index.name} already exists on ${index.table}`)
+    return
+  }
+
+  logger.info(`${loggerPrefix} adding index ${index.name} on ${index.table}(${index.fields.join(', ')})`)
+  await queryInterface.addIndex(index.table, {
+    name: index.name,
+    fields: index.fields
+  })
+  logger.info(`${loggerPrefix} added index ${index.name}`)
+}
+
+async function removeIndexIfExists(queryInterface, logger, index) {
+  const existing = await queryInterface.showIndex(index.table)
+  if (!existing.some((i) => i.name === index.name)) {
+    logger.info(`${loggerPrefix} index ${index.name} does not exist on ${index.table}`)
+    return
+  }
+
+  logger.info(`${loggerPrefix} removing index ${index.name}`)
+  await queryInterface.removeIndex(index.table, index.name)
+  logger.info(`${loggerPrefix} removed index ${index.name}`)
+}
+
+module.exports = { up, down }
--- a/server/models/LibraryItem.js
+++ b/server/models/LibraryItem.js
@ -340,6 +340,15 @@ class LibraryItem extends Model {

    const shelves = []

+    const timed = async (loader) => {
+      const start = Date.now()
+      const payload = await loader()
+      return {
+        payload,
+        elapsedSeconds: ((Date.now() - start) / 1000).toFixed(2)
+      }
+    }
+
    // "Continue Listening" shelf
    const itemsInProgressPayload = await libraryFilters.getMediaItemsInProgress(library, user, include, limit, false)
    if (itemsInProgressPayload.items.length) {
@ -371,11 +380,18 @@ class LibraryItem extends Model {
    }
    Logger.debug(`Loaded ${itemsInProgressPayload.items.length} of ${itemsInProgressPayload.count} items for "Continue Listening/Reading" in ${((Date.now() - fullStart) / 1000).toFixed(2)}s`)

-    let start = Date.now()
    if (library.isBook) {
-      start = Date.now()
+      const [continueSeriesResult, mostRecentResult, seriesMostRecentResult, discoverResult, mediaFinishedResult, newestAuthorsResult] = await Promise.all([
+        timed(() => libraryFilters.getLibraryItemsContinueSeries(library, user, include, limit)),
+        timed(() => libraryFilters.getLibraryItemsMostRecentlyAdded(library, user, include, limit)),
+        timed(() => libraryFilters.getSeriesMostRecentlyAdded(library, user, include, 5)),
+        timed(() => libraryFilters.getLibraryItemsToDiscover(library, user, include, limit)),
+        timed(() => libraryFilters.getMediaFinished(library, user, include, limit)),
+        timed(() => libraryFilters.getNewestAuthors(library, user, limit))
+      ])
+
+      const continueSeriesPayload = continueSeriesResult.payload
      // "Continue Series" shelf
-      const continueSeriesPayload = await libraryFilters.getLibraryItemsContinueSeries(library, user, include, limit)
      if (continueSeriesPayload.libraryItems.length) {
        shelves.push({
          id: 'continue-series',
@ -386,42 +402,24 @@ class LibraryItem extends Model {
          total: continueSeriesPayload.count
        })
      }
-      Logger.debug(`Loaded ${continueSeriesPayload.libraryItems.length} of ${continueSeriesPayload.count} items for "Continue Series" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
-    } else if (library.isPodcast) {
-      // "Newest Episodes" shelf
-      const newestEpisodesPayload = await libraryFilters.getNewestPodcastEpisodes(library, user, limit)
-      if (newestEpisodesPayload.libraryItems.length) {
+      Logger.debug(`Loaded ${continueSeriesPayload.libraryItems.length} of ${continueSeriesPayload.count} items for "Continue Series" in ${continueSeriesResult.elapsedSeconds}s`)
+
+      const mostRecentPayload = mostRecentResult.payload
+      // "Recently Added" shelf
+      if (mostRecentPayload.libraryItems.length) {
        shelves.push({
-          id: 'newest-episodes',
-          label: 'Newest Episodes',
-          labelStringKey: 'LabelNewestEpisodes',
-          type: 'episode',
-          entities: newestEpisodesPayload.libraryItems,
-          total: newestEpisodesPayload.count
+          id: 'recently-added',
+          label: 'Recently Added',
+          labelStringKey: 'LabelRecentlyAdded',
+          type: library.mediaType,
+          entities: mostRecentPayload.libraryItems,
+          total: mostRecentPayload.count
        })
      }
-      Logger.debug(`Loaded ${newestEpisodesPayload.libraryItems.length} of ${newestEpisodesPayload.count} episodes for "Newest Episodes" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
-    }
+      Logger.debug(`Loaded ${mostRecentPayload.libraryItems.length} of ${mostRecentPayload.count} items for "Recently Added" in ${mostRecentResult.elapsedSeconds}s`)

-    start = Date.now()
-    // "Recently Added" shelf
-    const mostRecentPayload = await libraryFilters.getLibraryItemsMostRecentlyAdded(library, user, include, limit)
-    if (mostRecentPayload.libraryItems.length) {
-      shelves.push({
-        id: 'recently-added',
-        label: 'Recently Added',
-        labelStringKey: 'LabelRecentlyAdded',
-        type: library.mediaType,
-        entities: mostRecentPayload.libraryItems,
-        total: mostRecentPayload.count
-      })
-    }
-    Logger.debug(`Loaded ${mostRecentPayload.libraryItems.length} of ${mostRecentPayload.count} items for "Recently Added" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
-
-    if (library.isBook) {
-      start = Date.now()
+      const seriesMostRecentPayload = seriesMostRecentResult.payload
      // "Recent Series" shelf
-      const seriesMostRecentPayload = await libraryFilters.getSeriesMostRecentlyAdded(library, user, include, 5)
      if (seriesMostRecentPayload.series.length) {
        shelves.push({
          id: 'recent-series',
@ -432,11 +430,10 @@ class LibraryItem extends Model {
          total: seriesMostRecentPayload.count
        })
      }
-      Logger.debug(`Loaded ${seriesMostRecentPayload.series.length} of ${seriesMostRecentPayload.count} series for "Recent Series" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
+      Logger.debug(`Loaded ${seriesMostRecentPayload.series.length} of ${seriesMostRecentPayload.count} series for "Recent Series" in ${seriesMostRecentResult.elapsedSeconds}s`)

-      start = Date.now()
+      const discoverLibraryItemsPayload = discoverResult.payload
      // "Discover" shelf
-      const discoverLibraryItemsPayload = await libraryFilters.getLibraryItemsToDiscover(library, user, include, limit)
      if (discoverLibraryItemsPayload.libraryItems.length) {
        shelves.push({
          id: 'discover',
@ -447,45 +444,41 @@ class LibraryItem extends Model {
          total: discoverLibraryItemsPayload.count
        })
      }
-      Logger.debug(`Loaded ${discoverLibraryItemsPayload.libraryItems.length} of ${discoverLibraryItemsPayload.count} items for "Discover" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
-    }
+      Logger.debug(`Loaded ${discoverLibraryItemsPayload.libraryItems.length} of ${discoverLibraryItemsPayload.count} items for "Discover" in ${discoverResult.elapsedSeconds}s`)

-    start = Date.now()
-    // "Listen Again" shelf
-    const mediaFinishedPayload = await libraryFilters.getMediaFinished(library, user, include, limit)
-    if (mediaFinishedPayload.items.length) {
-      const ebookOnlyItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.ebookFormat && !li.media.numTracks)
-      const audioItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.numTracks || li.mediaType === 'podcast')
+      const mediaFinishedPayload = mediaFinishedResult.payload
+      // "Listen Again" shelf
+      if (mediaFinishedPayload.items.length) {
+        const ebookOnlyItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.ebookFormat && !li.media.numTracks)
+        const audioItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.numTracks || li.mediaType === 'podcast')

-      if (audioItemsInProgress.length) {
-        shelves.push({
-          id: 'listen-again',
-          label: 'Listen Again',
-          labelStringKey: 'LabelListenAgain',
-          type: library.isPodcast ? 'episode' : 'book',
-          entities: audioItemsInProgress,
-          total: mediaFinishedPayload.count
-        })
+        if (audioItemsInProgress.length) {
+          shelves.push({
+            id: 'listen-again',
+            label: 'Listen Again',
+            labelStringKey: 'LabelListenAgain',
+            type: library.isPodcast ? 'episode' : 'book',
+            entities: audioItemsInProgress,
+            total: mediaFinishedPayload.count
+          })
+        }
+
+        if (ebookOnlyItemsInProgress.length) {
+          // "Read Again" shelf
+          shelves.push({
+            id: 'read-again',
+            label: 'Read Again',
+            labelStringKey: 'LabelReadAgain',
+            type: 'book',
+            entities: ebookOnlyItemsInProgress,
+            total: mediaFinishedPayload.count
+          })
+        }
      }
+      Logger.debug(`Loaded ${mediaFinishedPayload.items.length} of ${mediaFinishedPayload.count} items for "Listen/Read Again" in ${mediaFinishedResult.elapsedSeconds}s`)

-      // "Read Again" shelf
-      if (ebookOnlyItemsInProgress.length) {
-        shelves.push({
-          id: 'read-again',
-          label: 'Read Again',
-          labelStringKey: 'LabelReadAgain',
-          type: 'book',
-          entities: ebookOnlyItemsInProgress,
-          total: mediaFinishedPayload.count
-        })
-      }
-    }
-    Logger.debug(`Loaded ${mediaFinishedPayload.items.length} of ${mediaFinishedPayload.count} items for "Listen/Read Again" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
-
-    if (library.isBook) {
-      start = Date.now()
+      const newestAuthorsPayload = newestAuthorsResult.payload
      // "Newest Authors" shelf
-      const newestAuthorsPayload = await libraryFilters.getNewestAuthors(library, user, limit)
      if (newestAuthorsPayload.authors.length) {
        shelves.push({
          id: 'newest-authors',
@ -496,7 +489,72 @@ class LibraryItem extends Model {
          total: newestAuthorsPayload.count
        })
      }
-      Logger.debug(`Loaded ${newestAuthorsPayload.authors.length} of ${newestAuthorsPayload.count} authors for "Newest Authors" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
+      Logger.debug(`Loaded ${newestAuthorsPayload.authors.length} of ${newestAuthorsPayload.count} authors for "Newest Authors" in ${newestAuthorsResult.elapsedSeconds}s`)
+    } else if (library.isPodcast) {
+      const [newestEpisodesResult, mostRecentResult, mediaFinishedResult] = await Promise.all([
+        timed(() => libraryFilters.getNewestPodcastEpisodes(library, user, limit)),
+        timed(() => libraryFilters.getLibraryItemsMostRecentlyAdded(library, user, include, limit)),
+        timed(() => libraryFilters.getMediaFinished(library, user, include, limit))
+      ])
+
+      const newestEpisodesPayload = newestEpisodesResult.payload
+      // "Newest Episodes" shelf
+      if (newestEpisodesPayload.libraryItems.length) {
+        shelves.push({
+          id: 'newest-episodes',
+          label: 'Newest Episodes',
+          labelStringKey: 'LabelNewestEpisodes',
+          type: 'episode',
+          entities: newestEpisodesPayload.libraryItems,
+          total: newestEpisodesPayload.count
+        })
+      }
+      Logger.debug(`Loaded ${newestEpisodesPayload.libraryItems.length} of ${newestEpisodesPayload.count} episodes for "Newest Episodes" in ${newestEpisodesResult.elapsedSeconds}s`)
+
+      const mostRecentPayload = mostRecentResult.payload
+      // "Recently Added" shelf
+      if (mostRecentPayload.libraryItems.length) {
+        shelves.push({
+          id: 'recently-added',
+          label: 'Recently Added',
+          labelStringKey: 'LabelRecentlyAdded',
+          type: library.mediaType,
+          entities: mostRecentPayload.libraryItems,
+          total: mostRecentPayload.count
+        })
+      }
+      Logger.debug(`Loaded ${mostRecentPayload.libraryItems.length} of ${mostRecentPayload.count} items for "Recently Added" in ${mostRecentResult.elapsedSeconds}s`)
+
+      const mediaFinishedPayload = mediaFinishedResult.payload
+      // "Listen Again" shelf
+      if (mediaFinishedPayload.items.length) {
+        const ebookOnlyItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.ebookFormat && !li.media.numTracks)
+        const audioItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.numTracks || li.mediaType === 'podcast')
+
+        if (audioItemsInProgress.length) {
+          shelves.push({
+            id: 'listen-again',
+            label: 'Listen Again',
+            labelStringKey: 'LabelListenAgain',
+            type: 'episode',
+            entities: audioItemsInProgress,
+            total: mediaFinishedPayload.count
+          })
+        }
+
+        if (ebookOnlyItemsInProgress.length) {
+          // "Read Again" shelf
+          shelves.push({
+            id: 'read-again',
+            label: 'Read Again',
+            labelStringKey: 'LabelReadAgain',
+            type: 'book',
+            entities: ebookOnlyItemsInProgress,
+            total: mediaFinishedPayload.count
+          })
+        }
+      }
+      Logger.debug(`Loaded ${mediaFinishedPayload.items.length} of ${mediaFinishedPayload.count} items for "Listen/Read Again" in ${mediaFinishedResult.elapsedSeconds}s`)
    }

    Logger.debug(`Loaded ${shelves.length} personalized shelves in ${((Date.now() - fullStart) / 1000).toFixed(2)}s`)
--- a/server/utils/queries/libraryItemsBookFilters.js
+++ b/server/utils/queries/libraryItemsBookFilters.js
@ -888,28 +888,79 @@ module.exports = {
      })
    }

-    // Step 2: Get books not started and not in a series OR is the first book of a series not started (ordered randomly)
-    const { rows: books, count } = await Database.bookModel.findAndCountAll({
-      where: [
-        {
-          '$mediaProgresses.isFinished$': {
-            [Sequelize.Op.or]: [null, 0]
-          },
-          '$mediaProgresses.currentTime$': {
-            [Sequelize.Op.or]: [null, 0]
-          },
-          [Sequelize.Op.or]: [
-            Sequelize.where(Sequelize.literal(`(SELECT COUNT(*) FROM bookSeries bs where bs.bookId = book.id)`), 0),
-            {
-              id: {
-                [Sequelize.Op.in]: booksFromSeriesToInclude
-              }
-            }
-          ]
+    const discoverWhere = [
+      {
+        '$mediaProgresses.isFinished$': {
+          [Sequelize.Op.or]: [null, 0]
        },
-        ...userPermissionBookWhere.bookWhere
-      ],
+        '$mediaProgresses.currentTime$': {
+          [Sequelize.Op.or]: [null, 0]
+        },
+        [Sequelize.Op.or]: [
+          Sequelize.where(Sequelize.literal(`(SELECT COUNT(*) FROM bookSeries bs where bs.bookId = book.id)`), 0),
+          {
+            id: {
+              [Sequelize.Op.in]: booksFromSeriesToInclude
+            }
+          }
+        ]
+      },
+      ...userPermissionBookWhere.bookWhere
+    ]
+
+    const baseDiscoverInclude = [
+      {
+        model: Database.libraryItemModel,
+        where: {
+          libraryId
+        }
+      },
+      {
+        model: Database.mediaProgressModel,
+        where: {
+          userId: user.id
+        },
+        required: false
+      }
+    ]
+
+    // Step 2a: Count with lightweight includes only
+    const count = await Database.bookModel.count({
+      where: discoverWhere,
      replacements: userPermissionBookWhere.replacements,
+      include: baseDiscoverInclude,
+      distinct: true,
+      col: 'id',
+      subQuery: false
+    })
+
+    // Step 2b: Select random IDs with lightweight includes only
+    const randomSelection = await Database.bookModel.findAll({
+      attributes: ['id'],
+      where: discoverWhere,
+      replacements: userPermissionBookWhere.replacements,
+      include: baseDiscoverInclude,
+      subQuery: false,
+      distinct: true,
+      limit,
+      order: Database.sequelize.random()
+    })
+
+    const selectedIds = randomSelection.map((b) => b.id).filter(Boolean)
+    if (!selectedIds.length) {
+      return {
+        libraryItems: [],
+        count
+      }
+    }
+
+    // Step 2c: Hydrate selected IDs with full metadata for API response
+    const books = await Database.bookModel.findAll({
+      where: {
+        id: {
+          [Sequelize.Op.in]: selectedIds
+        }
+      },
      include: [
        {
          model: Database.libraryItemModel,
@ -918,13 +969,6 @@ module.exports = {
          },
          include: libraryItemIncludes
        },
-        {
-          model: Database.mediaProgressModel,
-          where: {
-            userId: user.id
-          },
-          required: false
-        },
        {
          model: Database.bookAuthorModel,
          attributes: ['authorId'],
@ -942,14 +986,14 @@ module.exports = {
          separate: true
        }
      ],
-      subQuery: false,
-      distinct: true,
-      limit,
-      order: Database.sequelize.random()
+      subQuery: false
    })

+    const booksById = new Map(books.map((b) => [b.id, b]))
+    const orderedBooks = selectedIds.map((id) => booksById.get(id)).filter(Boolean)
+
    // Step 3: Map books to library items
-    const libraryItems = books.map((bookExpanded) => {
+    const libraryItems = orderedBooks.map((bookExpanded) => {
      const libraryItem = bookExpanded.libraryItem
      const book = bookExpanded
      delete book.libraryItem