Speed up personalized shelves and reduce search payload size

server/models/LibraryItem.js

@@ -340,6 +340,15 @@ class LibraryItem extends Model {
 
     const shelves = []
 
+    const timed = async (loader) => {
+      const start = Date.now()
+      const payload = await loader()
+      return {
+        payload,
+        elapsedSeconds: ((Date.now() - start) / 1000).toFixed(2)
+      }
+    }
+
     // "Continue Listening" shelf
     const itemsInProgressPayload = await libraryFilters.getMediaItemsInProgress(library, user, include, limit, false)
     if (itemsInProgressPayload.items.length) {
@@ -371,11 +380,18 @@ class LibraryItem extends Model {
     }
     Logger.debug(`Loaded ${itemsInProgressPayload.items.length} of ${itemsInProgressPayload.count} items for "Continue Listening/Reading" in ${((Date.now() - fullStart) / 1000).toFixed(2)}s`)
 
-    let start = Date.now()
     if (library.isBook) {
-      start = Date.now()
+      const [continueSeriesResult, mostRecentResult, seriesMostRecentResult, discoverResult, mediaFinishedResult, newestAuthorsResult] = await Promise.all([
+        timed(() => libraryFilters.getLibraryItemsContinueSeries(library, user, include, limit)),
+        timed(() => libraryFilters.getLibraryItemsMostRecentlyAdded(library, user, include, limit)),
+        timed(() => libraryFilters.getSeriesMostRecentlyAdded(library, user, include, 5)),
+        timed(() => libraryFilters.getLibraryItemsToDiscover(library, user, include, limit)),
+        timed(() => libraryFilters.getMediaFinished(library, user, include, limit)),
+        timed(() => libraryFilters.getNewestAuthors(library, user, limit))
+      ])
+
+      const continueSeriesPayload = continueSeriesResult.payload
       // "Continue Series" shelf
-      const continueSeriesPayload = await libraryFilters.getLibraryItemsContinueSeries(library, user, include, limit)
       if (continueSeriesPayload.libraryItems.length) {
         shelves.push({
           id: 'continue-series',
@@ -386,42 +402,24 @@ class LibraryItem extends Model {
           total: continueSeriesPayload.count
         })
       }
-      Logger.debug(`Loaded ${continueSeriesPayload.libraryItems.length} of ${continueSeriesPayload.count} items for "Continue Series" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
+      Logger.debug(`Loaded ${continueSeriesPayload.libraryItems.length} of ${continueSeriesPayload.count} items for "Continue Series" in ${continueSeriesResult.elapsedSeconds}s`)
-    } else if (library.isPodcast) {
+
-      // "Newest Episodes" shelf
+      const mostRecentPayload = mostRecentResult.payload
-      const newestEpisodesPayload = await libraryFilters.getNewestPodcastEpisodes(library, user, limit)
+      // "Recently Added" shelf
-      if (newestEpisodesPayload.libraryItems.length) {
+      if (mostRecentPayload.libraryItems.length) {
         shelves.push({
-          id: 'newest-episodes',
+          id: 'recently-added',
-          label: 'Newest Episodes',
+          label: 'Recently Added',
-          labelStringKey: 'LabelNewestEpisodes',
+          labelStringKey: 'LabelRecentlyAdded',
-          type: 'episode',
+          type: library.mediaType,
-          entities: newestEpisodesPayload.libraryItems,
+          entities: mostRecentPayload.libraryItems,
-          total: newestEpisodesPayload.count
+          total: mostRecentPayload.count
         })
       }
-      Logger.debug(`Loaded ${newestEpisodesPayload.libraryItems.length} of ${newestEpisodesPayload.count} episodes for "Newest Episodes" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
+      Logger.debug(`Loaded ${mostRecentPayload.libraryItems.length} of ${mostRecentPayload.count} items for "Recently Added" in ${mostRecentResult.elapsedSeconds}s`)
-    }
 
-    start = Date.now()
+      const seriesMostRecentPayload = seriesMostRecentResult.payload
-    // "Recently Added" shelf
-    const mostRecentPayload = await libraryFilters.getLibraryItemsMostRecentlyAdded(library, user, include, limit)
-    if (mostRecentPayload.libraryItems.length) {
-      shelves.push({
-        id: 'recently-added',
-        label: 'Recently Added',
-        labelStringKey: 'LabelRecentlyAdded',
-        type: library.mediaType,
-        entities: mostRecentPayload.libraryItems,
-        total: mostRecentPayload.count
-      })
-    }
-    Logger.debug(`Loaded ${mostRecentPayload.libraryItems.length} of ${mostRecentPayload.count} items for "Recently Added" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
-
-    if (library.isBook) {
-      start = Date.now()
       // "Recent Series" shelf
-      const seriesMostRecentPayload = await libraryFilters.getSeriesMostRecentlyAdded(library, user, include, 5)
       if (seriesMostRecentPayload.series.length) {
         shelves.push({
           id: 'recent-series',
@@ -432,11 +430,10 @@ class LibraryItem extends Model {
           total: seriesMostRecentPayload.count
         })
       }
-      Logger.debug(`Loaded ${seriesMostRecentPayload.series.length} of ${seriesMostRecentPayload.count} series for "Recent Series" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
+      Logger.debug(`Loaded ${seriesMostRecentPayload.series.length} of ${seriesMostRecentPayload.count} series for "Recent Series" in ${seriesMostRecentResult.elapsedSeconds}s`)
 
-      start = Date.now()
+      const discoverLibraryItemsPayload = discoverResult.payload
       // "Discover" shelf
-      const discoverLibraryItemsPayload = await libraryFilters.getLibraryItemsToDiscover(library, user, include, limit)
       if (discoverLibraryItemsPayload.libraryItems.length) {
         shelves.push({
           id: 'discover',
@@ -447,45 +444,41 @@ class LibraryItem extends Model {
           total: discoverLibraryItemsPayload.count
         })
       }
-      Logger.debug(`Loaded ${discoverLibraryItemsPayload.libraryItems.length} of ${discoverLibraryItemsPayload.count} items for "Discover" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
+      Logger.debug(`Loaded ${discoverLibraryItemsPayload.libraryItems.length} of ${discoverLibraryItemsPayload.count} items for "Discover" in ${discoverResult.elapsedSeconds}s`)
-    }
 
-    start = Date.now()
+      const mediaFinishedPayload = mediaFinishedResult.payload
-    // "Listen Again" shelf
+      // "Listen Again" shelf
-    const mediaFinishedPayload = await libraryFilters.getMediaFinished(library, user, include, limit)
+      if (mediaFinishedPayload.items.length) {
-    if (mediaFinishedPayload.items.length) {
+        const ebookOnlyItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.ebookFormat && !li.media.numTracks)
-      const ebookOnlyItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.ebookFormat && !li.media.numTracks)
+        const audioItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.numTracks || li.mediaType === 'podcast')
-      const audioItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.numTracks || li.mediaType === 'podcast')
 
-      if (audioItemsInProgress.length) {
+        if (audioItemsInProgress.length) {
-        shelves.push({
+          shelves.push({
-          id: 'listen-again',
+            id: 'listen-again',
-          label: 'Listen Again',
+            label: 'Listen Again',
-          labelStringKey: 'LabelListenAgain',
+            labelStringKey: 'LabelListenAgain',
-          type: library.isPodcast ? 'episode' : 'book',
+            type: library.isPodcast ? 'episode' : 'book',
-          entities: audioItemsInProgress,
+            entities: audioItemsInProgress,
-          total: mediaFinishedPayload.count
+            total: mediaFinishedPayload.count
-        })
+          })
+        }
+
+        if (ebookOnlyItemsInProgress.length) {
+          // "Read Again" shelf
+          shelves.push({
+            id: 'read-again',
+            label: 'Read Again',
+            labelStringKey: 'LabelReadAgain',
+            type: 'book',
+            entities: ebookOnlyItemsInProgress,
+            total: mediaFinishedPayload.count
+          })
+        }
       }
+      Logger.debug(`Loaded ${mediaFinishedPayload.items.length} of ${mediaFinishedPayload.count} items for "Listen/Read Again" in ${mediaFinishedResult.elapsedSeconds}s`)
 
-      // "Read Again" shelf
+      const newestAuthorsPayload = newestAuthorsResult.payload
-      if (ebookOnlyItemsInProgress.length) {
-        shelves.push({
-          id: 'read-again',
-          label: 'Read Again',
-          labelStringKey: 'LabelReadAgain',
-          type: 'book',
-          entities: ebookOnlyItemsInProgress,
-          total: mediaFinishedPayload.count
-        })
-      }
-    }
-    Logger.debug(`Loaded ${mediaFinishedPayload.items.length} of ${mediaFinishedPayload.count} items for "Listen/Read Again" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
-
-    if (library.isBook) {
-      start = Date.now()
       // "Newest Authors" shelf
-      const newestAuthorsPayload = await libraryFilters.getNewestAuthors(library, user, limit)
       if (newestAuthorsPayload.authors.length) {
         shelves.push({
           id: 'newest-authors',
@@ -496,7 +489,72 @@ class LibraryItem extends Model {
           total: newestAuthorsPayload.count
         })
       }
-      Logger.debug(`Loaded ${newestAuthorsPayload.authors.length} of ${newestAuthorsPayload.count} authors for "Newest Authors" in ${((Date.now() - start) / 1000).toFixed(2)}s`)
+      Logger.debug(`Loaded ${newestAuthorsPayload.authors.length} of ${newestAuthorsPayload.count} authors for "Newest Authors" in ${newestAuthorsResult.elapsedSeconds}s`)
+    } else if (library.isPodcast) {
+      const [newestEpisodesResult, mostRecentResult, mediaFinishedResult] = await Promise.all([
+        timed(() => libraryFilters.getNewestPodcastEpisodes(library, user, limit)),
+        timed(() => libraryFilters.getLibraryItemsMostRecentlyAdded(library, user, include, limit)),
+        timed(() => libraryFilters.getMediaFinished(library, user, include, limit))
+      ])
+
+      const newestEpisodesPayload = newestEpisodesResult.payload
+      // "Newest Episodes" shelf
+      if (newestEpisodesPayload.libraryItems.length) {
+        shelves.push({
+          id: 'newest-episodes',
+          label: 'Newest Episodes',
+          labelStringKey: 'LabelNewestEpisodes',
+          type: 'episode',
+          entities: newestEpisodesPayload.libraryItems,
+          total: newestEpisodesPayload.count
+        })
+      }
+      Logger.debug(`Loaded ${newestEpisodesPayload.libraryItems.length} of ${newestEpisodesPayload.count} episodes for "Newest Episodes" in ${newestEpisodesResult.elapsedSeconds}s`)
+
+      const mostRecentPayload = mostRecentResult.payload
+      // "Recently Added" shelf
+      if (mostRecentPayload.libraryItems.length) {
+        shelves.push({
+          id: 'recently-added',
+          label: 'Recently Added',
+          labelStringKey: 'LabelRecentlyAdded',
+          type: library.mediaType,
+          entities: mostRecentPayload.libraryItems,
+          total: mostRecentPayload.count
+        })
+      }
+      Logger.debug(`Loaded ${mostRecentPayload.libraryItems.length} of ${mostRecentPayload.count} items for "Recently Added" in ${mostRecentResult.elapsedSeconds}s`)
+
+      const mediaFinishedPayload = mediaFinishedResult.payload
+      // "Listen Again" shelf
+      if (mediaFinishedPayload.items.length) {
+        const ebookOnlyItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.ebookFormat && !li.media.numTracks)
+        const audioItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.numTracks || li.mediaType === 'podcast')
+
+        if (audioItemsInProgress.length) {
+          shelves.push({
+            id: 'listen-again',
+            label: 'Listen Again',
+            labelStringKey: 'LabelListenAgain',
+            type: 'episode',
+            entities: audioItemsInProgress,
+            total: mediaFinishedPayload.count
+          })
+        }
+
+        if (ebookOnlyItemsInProgress.length) {
+          // "Read Again" shelf
+          shelves.push({
+            id: 'read-again',
+            label: 'Read Again',
+            labelStringKey: 'LabelReadAgain',
+            type: 'book',
+            entities: ebookOnlyItemsInProgress,
+            total: mediaFinishedPayload.count
+          })
+        }
+      }
+      Logger.debug(`Loaded ${mediaFinishedPayload.items.length} of ${mediaFinishedPayload.count} items for "Listen/Read Again" in ${mediaFinishedResult.elapsedSeconds}s`)
     }
 
     Logger.debug(`Loaded ${shelves.length} personalized shelves in ${((Date.now() - fullStart) / 1000).toFixed(2)}s`)

server/utils/queries/libraryItemsBookFilters.js

@@ -888,28 +888,80 @@ module.exports = {
       })
     }
 
-    // Step 2: Get books not started and not in a series OR is the first book of a series not started (ordered randomly)
+    const discoverWhere = [
-    const { rows: books, count } = await Database.bookModel.findAndCountAll({
+      {
-      where: [
+        [Sequelize.Op.and]: [
-        {
+          Sequelize.where(
-          '$mediaProgresses.isFinished$': {
+            Sequelize.literal(
-            [Sequelize.Op.or]: [null, 0]
+              `(SELECT COUNT(*) FROM mediaProgresses mp WHERE mp.mediaItemId = book.id AND mp.userId = :userId AND (mp.isFinished = 1 OR mp.currentTime > 0))`
-          },
+            ),
-          '$mediaProgresses.currentTime$': {
+            0
-            [Sequelize.Op.or]: [null, 0]
+          )
-          },
+        ],
-          [Sequelize.Op.or]: [
+        [Sequelize.Op.or]: [
-            Sequelize.where(Sequelize.literal(`(SELECT COUNT(*) FROM bookSeries bs where bs.bookId = book.id)`), 0),
+          Sequelize.where(Sequelize.literal(`(SELECT COUNT(*) FROM bookSeries bs where bs.bookId = book.id)`), 0),
-            {
+          {
-              id: {
+            id: {
-                [Sequelize.Op.in]: booksFromSeriesToInclude
+              [Sequelize.Op.in]: booksFromSeriesToInclude
-              }
             }
-          ]
+          }
-        },
+        ]
-        ...userPermissionBookWhere.bookWhere
+      },
-      ],
+      ...userPermissionBookWhere.bookWhere
-      replacements: userPermissionBookWhere.replacements,
+    ]
+
+    const baseDiscoverInclude = [
+      {
+        model: Database.libraryItemModel,
+        where: {
+          libraryId
+        }
+      }
+    ]
+
+    // Step 2a: Count with lightweight includes only
+    const count = await Database.bookModel.count({
+      where: discoverWhere,
+      replacements: {
+        userId: user.id,
+        ...userPermissionBookWhere.replacements
+      },
+      include: baseDiscoverInclude,
+      distinct: true,
+      col: 'id',
+      subQuery: false
+    })
+
+    // Step 2b: Select random IDs with lightweight includes only
+    const randomSelection = await Database.bookModel.findAll({
+      attributes: ['id'],
+      where: discoverWhere,
+      replacements: {
+        userId: user.id,
+        ...userPermissionBookWhere.replacements
+      },
+      include: baseDiscoverInclude,
+      subQuery: false,
+      distinct: true,
+      limit,
+      order: Database.sequelize.random()
+    })
+
+    const selectedIds = randomSelection.map((b) => b.id).filter(Boolean)
+    if (!selectedIds.length) {
+      return {
+        libraryItems: [],
+        count
+      }
+    }
+
+    // Step 2c: Hydrate selected IDs with full metadata for API response
+    const books = await Database.bookModel.findAll({
+      where: {
+        id: {
+          [Sequelize.Op.in]: selectedIds
+        }
+      },
       include: [
         {
           model: Database.libraryItemModel,
@@ -918,13 +970,6 @@ module.exports = {
           },
           include: libraryItemIncludes
         },
-        {
-          model: Database.mediaProgressModel,
-          where: {
-            userId: user.id
-          },
-          required: false
-        },
         {
           model: Database.bookAuthorModel,
           attributes: ['authorId'],
@@ -942,14 +987,14 @@ module.exports = {
           separate: true
         }
       ],
-      subQuery: false,
+      subQuery: false
-      distinct: true,
-      limit,
-      order: Database.sequelize.random()
     })
 
+    const booksById = new Map(books.map((b) => [b.id, b]))
+    const orderedBooks = selectedIds.map((id) => booksById.get(id)).filter(Boolean)
+
     // Step 3: Map books to library items
-    const libraryItems = books.map((bookExpanded) => {
+    const libraryItems = orderedBooks.map((bookExpanded) => {
       const libraryItem = bookExpanded.libraryItem
       const book = bookExpanded
       delete book.libraryItem
@@ -1122,7 +1167,7 @@ module.exports = {
 
       libraryItem.media = book
       itemMatches.push({
-        libraryItem: libraryItem.toOldJSONExpanded()
+        libraryItem: libraryItem.toOldJSONMinified()
       })
     }
 

server/utils/queries/libraryItemsPodcastFilters.js

@@ -410,7 +410,7 @@ module.exports = {
       libraryItem.media = podcast
       libraryItem.media.podcastEpisodes = []
       itemMatches.push({
-        libraryItem: libraryItem.toOldJSONExpanded()
+        libraryItem: libraryItem.toOldJSONMinified()
       })
     }
 
@@ -444,7 +444,7 @@ module.exports = {
       libraryItem.media = episode.podcast
       libraryItem.media.podcastEpisodes = []
       const oldPodcastEpisodeJson = episode.toOldJSONExpanded(libraryItem.id)
-      const libraryItemJson = libraryItem.toOldJSONExpanded()
+      const libraryItemJson = libraryItem.toOldJSONMinified()
       libraryItemJson.recentEpisode = oldPodcastEpisodeJson
       episodeMatches.push({
         libraryItem: libraryItemJson