feat: split book browse rows from counts

server/utils/queries/libraryBrowseStrategy.js

@@ -60,6 +60,7 @@ function getFamily({ mediaType, sortBy, filterGroup, collapseseries }) {
 
 function getLibraryBrowseStrategy({ mediaType, sortBy, filterGroup, pageMode, collapseseries } = {}) {
   const normalizedSort = sortBy || 'media.metadata.title'
+  const paginationMode = getPaginationMode(pageMode, normalizedSort)
 
   if (normalizedSort === 'random') {
     return {
@@ -74,9 +75,9 @@ function getLibraryBrowseStrategy({ mediaType, sortBy, filterGroup, pageMode, co
 
   return {
     family: getFamily({ mediaType, sortBy: normalizedSort, filterGroup, collapseseries }),
-    paginationMode: getPaginationMode(pageMode, normalizedSort),
-    countMode: 'deferred-exact',
-    deepScrollAllowed: true,
+    paginationMode,
+    countMode: paginationMode === 'keyset' ? 'deferred-exact' : 'exact-on-initial-page',
+    deepScrollAllowed: paginationMode === 'keyset',
     tieBreaker: 'id',
     cursorKeys: getKeysetCursorKeys(normalizedSort)
   }

server/utils/queries/libraryItemsBookFilters.js

@@ -6,8 +6,107 @@ const authorFilters = require('./authorFilters')
 const ShareManager = require('../../managers/ShareManager')
 const { profile } = require('../profiler')
 const stringifySequelizeQuery = require('../stringifySequelizeQuery')
+const { getLibraryBrowseStrategy } = require('./libraryBrowseStrategy')
+const { loadBrowseCount } = require('./libraryBrowseCount')
+const { encodeBrowseCursor, decodeBrowseCursor } = require('./libraryBrowseCursor')
 const countCache = new Map()
 
+function getCursorFieldPath(key) {
+  if (['title', 'titleIgnorePrefix', 'authorNamesFirstLast', 'authorNamesLastFirst', 'createdAt', 'updatedAt'].includes(key)) {
+    return `$libraryItem.${key}$`
+  }
+
+  if (key === 'id') {
+    return '$libraryItem.id$'
+  }
+
+  if (key.startsWith('mediaProgresses.')) {
+    return `$${key}$`
+  }
+
+  return `$${key}$`
+}
+
+function getCursorValue(book, key) {
+  if (key === 'id') {
+    return book.libraryItem?.id || null
+  }
+
+  if (key.startsWith('mediaProgresses.')) {
+    const field = key.split('.')[1]
+    return book.mediaProgresses?.[0]?.[field] ?? null
+  }
+
+  if (book.libraryItem && Object.prototype.hasOwnProperty.call(book.libraryItem.dataValues || {}, key)) {
+    return book.libraryItem[key]
+  }
+
+  return book[key] ?? null
+}
+
+function buildBookCursorClause(cursor, cursorKeys, sortDesc) {
+  if (!cursor || !cursorKeys.length) return null
+
+  let decodedCursor
+  try {
+    decodedCursor = decodeBrowseCursor(cursor)
+  } catch (error) {
+    return null
+  }
+
+  if (decodedCursor.keys.join('|') !== cursorKeys.join('|')) {
+    throw new Error('Browse cursor keys do not match the requested sort')
+  }
+
+  const comparisonOperator = sortDesc ? Sequelize.Op.lt : Sequelize.Op.gt
+  const cursorConditions = decodedCursor.keys.map((key, index) => {
+    const condition = {}
+
+    for (let cursorIndex = 0; cursorIndex < index; cursorIndex++) {
+      condition[getCursorFieldPath(decodedCursor.keys[cursorIndex])] = decodedCursor.values[cursorIndex]
+    }
+
+    condition[getCursorFieldPath(key)] = {
+      [comparisonOperator]: decodedCursor.values[index]
+    }
+
+    return condition
+  })
+
+  return {
+    [Sequelize.Op.or]: cursorConditions
+  }
+}
+
+async function loadBookBrowseRows({ model, findOptions, limit, cursorClause }) {
+  const whereClauses = Array.isArray(findOptions.where) ? [...findOptions.where] : [findOptions.where]
+  const rowLimit = Number(limit) || null
+
+  return model.findAll({
+    ...findOptions,
+    where: cursorClause ? [...whereClauses, cursorClause] : whereClauses,
+    limit: rowLimit ? rowLimit + 1 : null
+  })
+}
+
+async function loadBookBrowseCount({ model, countOptions }) {
+  return model.count(countOptions)
+}
+
+function getNextBrowseCursor(books, limit, sortBy, sortDesc, cursorKeys) {
+  const rowLimit = Number(limit) || 0
+  if (!rowLimit || books.length <= rowLimit) return null
+
+  const lastBook = books[rowLimit - 1]
+
+  return encodeBrowseCursor({
+    sortBy,
+    desc: !!sortDesc,
+    keys: cursorKeys,
+    values: cursorKeys.map((key) => getCursorValue(lastBook, key))
+  })
+}
+
 module.exports = {
   /**
    * User permissions to restrict books for explicit content & tags
@@ -622,8 +721,53 @@ module.exports = {
       subQuery: false
     }
 
-    const findAndCountAll = process.env.QUERY_PROFILING ? profile(this.findAndCountAll) : this.findAndCountAll
-    const { rows: books, count } = await findAndCountAll(findOptions, limit, offset, !filterGroup && !userPermissionBookWhere.bookWhere.length)
+    const strategy = getLibraryBrowseStrategy({
+      mediaType: 'book',
+      sortBy,
+      filterGroup,
+      pageMode: browseRequestOptions?.pageMode,
+      collapseseries
+    })
+
+    let books = []
+    let count = 0
+    let nextCursor = null
+    let paginationMode = strategy.paginationMode
+    let countMode = strategy.countMode
+    let isCountDeferred = false
+
+    if (strategy.paginationMode === 'keyset') {
+      const countOptions = { ...findOptions }
+      delete countOptions.order
+
+      const cursorClause = buildBookCursorClause(browseRequestOptions?.cursor, strategy.cursorKeys, sortDesc)
+      const effectiveCountMode = browseRequestOptions?.cursor ? 'skip' : strategy.countMode
+      const [loadedBooks, countPayload] = await Promise.all([
+        loadBookBrowseRows({
+          model: Database.bookModel,
+          findOptions,
+          limit,
+          cursorClause
+        }),
+        loadBrowseCount({
+          mode: effectiveCountMode,
+          exactCountLoader: () => loadBookBrowseCount({ model: Database.bookModel, countOptions })
+        })
+      ])
+
+      nextCursor = getNextBrowseCursor(loadedBooks, limit, sortBy, sortDesc, strategy.cursorKeys)
+      books = Number(limit) ? loadedBooks.slice(0, Number(limit)) : loadedBooks
+      count = countPayload.total
+      countMode = effectiveCountMode === 'skip' ? 'skip' : strategy.countMode
+      isCountDeferred = countPayload.isDeferred
+    } else {
+      const findAndCountAll = process.env.QUERY_PROFILING ? profile(this.findAndCountAll) : this.findAndCountAll
+      const result = await findAndCountAll(findOptions, limit, offset, !filterGroup && !userPermissionBookWhere.bookWhere.length)
+      books = result.rows
+      count = result.count
+      countMode = 'exact-on-initial-page'
+      paginationMode = 'offset'
+    }
 
     const libraryItems = books.map((bookExpanded) => {
       const libraryItem = bookExpanded.libraryItem
@@ -683,7 +827,11 @@ module.exports = {
 
     return {
       libraryItems,
-      count
+      count,
+      nextCursor,
+      paginationMode,
+      countMode,
+      isCountDeferred
     }
   },
 

test/server/controllers/LibraryController.largeLibraryBrowse.test.js

@@ -200,4 +200,30 @@ describe('LibraryController large-library browse contract', () => {
       collapseseries: false
     })
   })
+
+  it('does not run exact count work or joined findAndCountAll again on follow-up keyset chunks', async () => {
+    const countStub = sinon.stub(Database.bookModel, 'count').resolves(123)
+    const findAllStub = sinon.stub(Database.bookModel, 'findAll').resolves([])
+    const findAndCountAllStub = sinon.stub(Database.bookModel, 'findAndCountAll').resolves({ rows: [], count: 123 })
+
+    const result = await libraryItemsBookFilters.getFilteredLibraryItems(
+      'lib_1',
+      { id: 'user_1', canAccessExplicitContent: true, accessAllTags: true },
+      null,
+      null,
+      'media.metadata.title',
+      false,
+      false,
+      [],
+      40,
+      0,
+      false,
+      { cursor: 'cursor-2', pageMode: 'endless' }
+    )
+
+    expect(result.count).to.equal(null)
+    expect(findAllStub.calledOnce).to.equal(true)
+    expect(countStub.called).to.equal(false)
+    expect(findAndCountAllStub.called).to.equal(false)
+  })
 })