groove/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2026-07-07 09:51:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

perf: bound collapsed-series anchor queries

Browse source
This commit is contained in:
Jonathan Finley 2026-03-13 22:57:35 -04:00
parent 272df76ac2
commit 3dc7ffedd7
2 changed files with 337 additions and 92 deletions
Download patch file Download diff file Expand all files Collapse all files

302
server/utils/queries/libraryItemsBookFilters.js
View file

@ -171,6 +171,189 @@ async function loadCollapsedSeriesWindow({ findOptions, countOptions, limit, off
return { books, count }
}
function getCollapsedSeriesSortContext(payload = {}) {
const collapsedSortBy = payload.sortBy || 'sequence'
const sortDesc = !!payload.sortDesc
const direction = sortDesc ? 'DESC' : 'ASC'
const titleColumn = Database.serverSettings.sortingIgnorePrefix ? 'titleIgnorePrefix' : 'title'
const nameColumn = Database.serverSettings.sortingIgnorePrefix ? 'nameIgnorePrefix' : 'name'
return {
collapsedSortBy,
sortDesc,
direction,
titleColumn,
nameColumn,
rawBookOrder: collapsedSortBy === 'media.metadata.title'
? `"plainSortValue" ${direction}, "libraryItemId" ASC`
: `"filterSequenceSort" ${direction} NULLS LAST, "plainSortValue" ${direction}, "libraryItemId" ASC`,
visibleRowOrder: collapsedSortBy === 'media.metadata.title'
? `"rowSortValue" ${direction}, "anchorLibraryItemId" ASC`
: `"filterSequenceSort" ${direction} NULLS LAST, "rowSortValue" ${direction}, "anchorLibraryItemId" ASC`
}
}
function buildCollapsedSeriesBaseQuery(libraryId, seriesId, userPermissionBookWhere, sortContext) {
const queryGenerator = Database.sequelize.dialect.queryGenerator
const sequenceSortExpr = dialectHelpers.getSafeSequenceCast(Database.getDialect(), '"bookSeries"."sequence"')
const plainSortExpr = `LOWER("libraryItem"."${sortContext.titleColumn}")`
const baseOptions = {
attributes: [
['id', 'bookId'],
[Sequelize.col('libraryItem.id'), 'libraryItemId'],
[Sequelize.col('bookSeries.sequence'), 'filterSequence'],
[Sequelize.literal(sequenceSortExpr), 'filterSequenceSort'],
[Sequelize.literal(plainSortExpr), 'plainSortValue']
],
where: userPermissionBookWhere.bookWhere,
replacements: userPermissionBookWhere.replacements,
include: [
{
model: Database.libraryItemModel,
required: true,
attributes: [],
where: {
libraryId
}
},
{
model: Database.bookSeriesModel,
required: true,
attributes: [],
where: {
seriesId
}
}
],
model: Database.bookModel,
subQuery: false
}
Database.bookModel._validateIncludedElements(baseOptions)
return queryGenerator.selectQuery(
Database.bookModel.getTableName(),
baseOptions,
Database.bookModel
).replace(/;\s*$/, '')
}
function buildCollapsedSeriesMetaQuery(baseQuery, hideSingleBookSeries) {
const activeSeriesFilter = hideSingleBookSeries ? 'WHERE sm."subseriesBookCount" > 1' : ''
return `WITH filtered_books AS (${baseQuery}),
subseries_members AS (
SELECT
fb."bookId",
fb."libraryItemId",
fb."filterSequenceSort",
fb."plainSortValue",
bs."seriesId" AS "subseriesId",
s.name AS "subseriesName",
s."nameIgnorePrefix" AS "subseriesNameIgnorePrefix",
COUNT(*) OVER (PARTITION BY bs."seriesId") AS "subseriesBookCount"
FROM filtered_books fb
JOIN "bookSeries" bs ON bs."bookId" = fb."bookId" AND bs."seriesId" <> :seriesId
JOIN series s ON s.id = bs."seriesId"
),
active_subseries AS (
SELECT DISTINCT sm."subseriesId"
FROM subseries_members sm
${activeSeriesFilter}
),
plain_rows AS (
SELECT fb."bookId"
FROM filtered_books fb
WHERE NOT EXISTS (
SELECT 1
FROM subseries_members sm
JOIN active_subseries act ON act."subseriesId" = sm."subseriesId"
WHERE sm."bookId" = fb."bookId"
)
),
collapsed_rows AS (
SELECT act."subseriesId"
FROM active_subseries act
)
SELECT
(SELECT COUNT(*) FROM filtered_books) AS "rawBookCount",
(SELECT COUNT(*) FROM plain_rows) AS "plainRowCount",
(SELECT COUNT(*) FROM collapsed_rows) AS "collapsedRowCount"`
}
function buildCollapsedSeriesRowsQuery(baseQuery, sortContext, hideSingleBookSeries, fallbackToPlain) {
const activeSeriesFilter = hideSingleBookSeries ? 'WHERE sm."subseriesBookCount" > 1' : ''
if (fallbackToPlain) {
return `WITH filtered_books AS (${baseQuery})
SELECT
'plain' AS "rowType",
fb."libraryItemId" AS "anchorLibraryItemId",
fb."bookId" AS "anchorBookId",
NULL AS "subseriesId"
FROM filtered_books fb
ORDER BY ${sortContext.rawBookOrder}
LIMIT :limit OFFSET :offset`
}
return `WITH filtered_books AS (${baseQuery}),
subseries_members AS (
SELECT
fb."bookId",
fb."libraryItemId",
fb."filterSequenceSort",
fb."plainSortValue",
bs."seriesId" AS "subseriesId",
s.name AS "subseriesName",
s."nameIgnorePrefix" AS "subseriesNameIgnorePrefix",
ROW_NUMBER() OVER (PARTITION BY bs."seriesId" ORDER BY ${sortContext.rawBookOrder}) AS "subseriesRank",
COUNT(*) OVER (PARTITION BY bs."seriesId") AS "subseriesBookCount"
FROM filtered_books fb
JOIN "bookSeries" bs ON bs."bookId" = fb."bookId" AND bs."seriesId" <> :seriesId
JOIN series s ON s.id = bs."seriesId"
),
active_subseries AS (
SELECT DISTINCT sm."subseriesId"
FROM subseries_members sm
${activeSeriesFilter}
),
collapsed_rows AS (
SELECT
'collapsed' AS "rowType",
sm."libraryItemId" AS "anchorLibraryItemId",
sm."bookId" AS "anchorBookId",
sm."subseriesId",
sm."filterSequenceSort",
LOWER(sm."subseries${sortContext.nameColumn === 'nameIgnorePrefix' ? 'NameIgnorePrefix' : 'Name'}") AS "rowSortValue"
FROM subseries_members sm
WHERE sm."subseriesRank" = 1
AND EXISTS (SELECT 1 FROM active_subseries act WHERE act."subseriesId" = sm."subseriesId")
),
plain_rows AS (
SELECT
'plain' AS "rowType",
fb."libraryItemId" AS "anchorLibraryItemId",
fb."bookId" AS "anchorBookId",
NULL AS "subseriesId",
fb."filterSequenceSort",
fb."plainSortValue" AS "rowSortValue"
FROM filtered_books fb
WHERE NOT EXISTS (
SELECT 1
FROM subseries_members sm
JOIN active_subseries act ON act."subseriesId" = sm."subseriesId"
WHERE sm."bookId" = fb."bookId"
)
)
SELECT *
FROM (
SELECT * FROM collapsed_rows
UNION ALL
SELECT * FROM plain_rows
) visible_rows
ORDER BY ${sortContext.visibleRowOrder}
LIMIT :limit OFFSET :offset`
}
function mapCollapsedSeriesBookToLibraryItem(bookExpanded) {
const libraryItem = bookExpanded.libraryItem
const book = bookExpanded
@ -190,7 +373,7 @@ function mapCollapsedSeriesBookToLibraryItem(bookExpanded) {
return libraryItem
}
function buildCollapsedSeriesView(libraryItems, seriesId, hideSingleBookSeries = false) {
function buildCollapsedSeriesView(libraryItems, seriesId, hideSingleBookSeries = false, skipSingletonCollapseFallback = false) {
const seriesEntriesById = {}
libraryItems.forEach((libraryItem) => {
@ -237,7 +420,7 @@ function buildCollapsedSeriesView(libraryItems, seriesId, hideSingleBookSeries =
}
})
if (visibleRows.length === 1 && visibleRows[0].type === 'collapsed') {
if (!skipSingletonCollapseFallback && visibleRows.length === 1 && visibleRows[0].type === 'collapsed') {
return {
visibleRows: libraryItems.map((libraryItem) => ({ type: 'plain', libraryItemId: libraryItem.id })),
seriesEntriesById: {}
@ -247,6 +430,15 @@ function buildCollapsedSeriesView(libraryItems, seriesId, hideSingleBookSeries =
return { visibleRows, seriesEntriesById }
}
function buildSelectedCollapsedSeriesEntries(libraryItems, seriesId, selectedSeriesIds, hideSingleBookSeries = false) {
if (!selectedSeriesIds.length) return {}
const visibleView = buildCollapsedSeriesView(libraryItems, seriesId, hideSingleBookSeries, true)
return Object.fromEntries(
Object.entries(visibleView.seriesEntriesById).filter(([selectedSeriesId]) => selectedSeriesIds.includes(selectedSeriesId))
)
}
function getNextBrowseCursor(books, limit, sortBy, sortDesc, cursorKeys) {
const rowLimit = Number(limit) || 0
if (!rowLimit || books.length <= rowLimit) return null
@ -1049,52 +1241,80 @@ module.exports = {
subQuery: false
}
const batchLimit = Math.max(Number(window.limit) || 0, 1)
const allLibraryItems = []
let rawOffset = 0
while (true) {
const { books } = await loadCollapsedSeriesWindow({
findOptions,
countOptions: null,
limit: batchLimit,
offset: rawOffset
})
if (!books.length) break
allLibraryItems.push(
...books
.map((bookExpanded) => mapCollapsedSeriesBookToLibraryItem(bookExpanded))
.filter((libraryItem) => user?.checkCanAccessLibraryItem ? user.checkCanAccessLibraryItem(libraryItem) : true)
)
rawOffset += books.length
if (books.length < batchLimit) break
const sortContext = getCollapsedSeriesSortContext(payload)
const baseQuery = buildCollapsedSeriesBaseQuery(libraryId, seriesId, userPermissionBookWhere, sortContext)
const replacements = {
...userPermissionBookWhere.replacements,
seriesId,
limit: Number(window.limit) || null,
offset: Number(window.offset) || 0
}
const { visibleRows, seriesEntriesById } = buildCollapsedSeriesView(allLibraryItems, seriesId, payload.hideSingleBookSeries)
const visibleOffset = Number(window.offset) || 0
const visibleLimit = Number(window.limit) || 0
const selectedRows = visibleLimit ? visibleRows.slice(visibleOffset, visibleOffset + visibleLimit) : visibleRows
const metaResult = await Database.sequelize.query(buildCollapsedSeriesMetaQuery(baseQuery, payload.hideSingleBookSeries), {
replacements,
type: Database.sequelize.QueryTypes.SELECT
})
const metaRows = Array.isArray(metaResult[0]) ? metaResult[0] : metaResult
const meta = Array.isArray(metaRows) ? metaRows[0] : metaRows
const fallbackToPlain = Number(meta?.plainRowCount || 0) === 0 && Number(meta?.collapsedRowCount || 0) === 1
const selectedRowsResult = await Database.sequelize.query(buildCollapsedSeriesRowsQuery(baseQuery, sortContext, payload.hideSingleBookSeries, fallbackToPlain), {
replacements,
type: Database.sequelize.QueryTypes.SELECT
})
const visibleRows = Array.isArray(selectedRowsResult[0]) ? selectedRowsResult[0] : selectedRowsResult
const selectedPlainBookIds = visibleRows.filter((row) => row.rowType === 'plain').map((row) => row.anchorBookId)
const selectedSeriesIds = visibleRows.filter((row) => row.rowType === 'collapsed').map((row) => row.subseriesId)
if (!selectedPlainBookIds.length && !selectedSeriesIds.length) {
return {
libraryItems: [],
count: fallbackToPlain ? Number(meta?.rawBookCount || 0) : Number(meta?.plainRowCount || 0) + Number(meta?.collapsedRowCount || 0)
}
}
const hydrateWhere = []
if (selectedPlainBookIds.length) {
hydrateWhere.push({ id: selectedPlainBookIds })
}
if (selectedSeriesIds.length) {
hydrateWhere.push(existsLiteral(`SELECT 1 FROM "bookSeries" collapseSeries WHERE collapseSeries."bookId" = book.id AND collapseSeries."seriesId" IN (:selectedCollapsedSeriesIds)`))
}
const hydratedBooks = await Database.bookModel.findAll({
...findOptions,
where: [
userPermissionBookWhere.bookWhere,
{
[Sequelize.Op.or]: hydrateWhere
}
],
replacements: {
...userPermissionBookWhere.replacements,
selectedCollapsedSeriesIds: selectedSeriesIds
}
})
const allLibraryItems = hydratedBooks
.map((bookExpanded) => mapCollapsedSeriesBookToLibraryItem(bookExpanded))
.filter((libraryItem) => user?.checkCanAccessLibraryItem ? user.checkCanAccessLibraryItem(libraryItem) : true)
const selectedSeriesEntries = buildSelectedCollapsedSeriesEntries(allLibraryItems, seriesId, selectedSeriesIds, payload.hideSingleBookSeries)
const libraryItemsById = new Map(allLibraryItems.map((libraryItem) => [libraryItem.id, libraryItem]))
const libraryItems = selectedRows
.map((row) => {
const libraryItem = libraryItemsById.get(row.libraryItemId)
if (!libraryItem) return null
if (row.type === 'collapsed') {
return Object.assign(Object.create(Object.getPrototypeOf(libraryItem)), libraryItem, {
collapsedSeries: seriesEntriesById[row.seriesId]
})
}
return libraryItem
})
.filter(Boolean)
const libraryItems = visibleRows.map((row) => {
const libraryItem = libraryItemsById.get(row.anchorLibraryItemId)
if (!libraryItem) return null
if (row.rowType === 'collapsed') {
return Object.assign(Object.create(Object.getPrototypeOf(libraryItem)), libraryItem, {
collapsedSeries: selectedSeriesEntries[row.subseriesId]
})
}
return libraryItem
}).filter(Boolean)
return {
libraryItems,
count: visibleRows.length
count: fallbackToPlain ? Number(meta?.rawBookCount || 0) : Number(meta?.plainRowCount || 0) + Number(meta?.collapsedRowCount || 0)
}
},

127
test/server/controllers/LibraryController.largeLibraryBrowse.test.js
View file

@ -276,6 +276,9 @@ describe('LibraryController large-library browse contract', () => {
it('returns only the requested collapsed-series window without loading the full series graph', async () => {
const findByPkStub = sinon.stub(Database.seriesModel, 'findByPk').resolves({ books: [] })
const queryStub = sinon.stub(Database.sequelize, 'query')
queryStub.onCall(0).resolves([{ rawBookCount: 0, plainRowCount: 0, collapsedRowCount: 0 }])
queryStub.onCall(1).resolves([])
const findAllStub = sinon.stub(Database.bookModel, 'findAll').resolves([])
const req = {
query: {
@ -292,13 +295,15 @@ describe('LibraryController large-library browse contract', () => {
await LibraryController.getLibraryItems(req, res)
expect(findByPkStub.called).to.equal(false)
expect(findAllStub.calledOnce).to.equal(true)
expect(findAllStub.firstCall.args[0].limit).to.equal(20)
expect(findAllStub.firstCall.args[0].offset || 0).to.equal(0)
expect(queryStub.calledTwice).to.equal(true)
expect(findAllStub.called).to.equal(false)
})
it('uses bounded batch query options for collapsed-series follow-up windows', async () => {
const findByPkStub = sinon.stub(Database.seriesModel, 'findByPk').resolves({ books: [] })
const queryStub = sinon.stub(Database.sequelize, 'query')
queryStub.onCall(0).resolves([{ rawBookCount: 0, plainRowCount: 0, collapsedRowCount: 0 }])
queryStub.onCall(1).resolves([])
const findAllStub = sinon.stub(Database.bookModel, 'findAll').resolves([])
const req = {
query: {
@ -317,9 +322,8 @@ describe('LibraryController large-library browse contract', () => {
await LibraryController.getLibraryItems(req, res)
expect(findByPkStub.called).to.equal(false)
expect(findAllStub.calledOnce).to.equal(true)
expect(findAllStub.firstCall.args[0].limit).to.equal(20)
expect(findAllStub.firstCall.args[0].offset).to.equal(0)
expect(queryStub.calledTwice).to.equal(true)
expect(findAllStub.called).to.equal(false)
})
it('restores collapsed-series payload fields and preserves rssfeed serialization', async () => {
@ -367,8 +371,10 @@ describe('LibraryController large-library browse contract', () => {
})
it('uses a deterministic default collapsed-series order when sort is omitted', async () => {
const queryStub = sinon.stub(Database.sequelize, 'query')
queryStub.onCall(0).resolves([{ rawBookCount: 0, plainRowCount: 0, collapsedRowCount: 0 }])
queryStub.onCall(1).resolves([])
const findAllStub = sinon.stub(Database.bookModel, 'findAll').resolves([])
sinon.stub(Database.bookModel, 'count').resolves(0)
await libraryItemsBookFilters.getCollapsedSeriesWindow(
'lib_1',
@ -379,9 +385,9 @@ describe('LibraryController large-library browse contract', () => {
{ limit: 20, offset: 0 }
)
expect(findAllStub.firstCall.args[0].order).to.have.length.greaterThan(0)
expect(findAllStub.firstCall.args[0].order[0][0].val || findAllStub.firstCall.args[0].order[0][0]).to.include('bookSeries')
expect(findAllStub.firstCall.args[0].order[0][0].val || findAllStub.firstCall.args[0].order[0][0]).to.include('sequence')
expect(queryStub.calledTwice).to.equal(true)
expect(findAllStub.called).to.equal(false)
expect(queryStub.secondCall.args[0]).to.include('filterSequenceSort')
})
it('paginates collapsed-series browse by visible rows instead of raw books', async () => {
@ -400,18 +406,12 @@ describe('LibraryController large-library browse contract', () => {
bookAuthors: []
})
const findAllStub = sinon.stub(Database.bookModel, 'findAll')
sinon.stub(Database.bookModel, 'count').resolves(3)
findAllStub.onCall(0).resolves([
makeBook({ libraryItemId: 'li_1', filterSequence: '1', subseriesId: 'sub_1', subseriesName: 'Subseries 1' })
])
findAllStub.onCall(1).resolves([
makeBook({ libraryItemId: 'li_2', filterSequence: '2', subseriesId: 'sub_1', subseriesName: 'Subseries 1' })
])
findAllStub.onCall(2).resolves([
const queryStub = sinon.stub(Database.sequelize, 'query')
queryStub.onCall(0).resolves([{ rawBookCount: 3, plainRowCount: 1, collapsedRowCount: 1 }])
queryStub.onCall(1).resolves([{ rowType: 'plain', anchorBookId: 'book_3', anchorLibraryItemId: 'li_3', subseriesId: null }])
const findAllStub = sinon.stub(Database.bookModel, 'findAll').resolves([
makeBook({ libraryItemId: 'li_3', filterSequence: '3' })
])
findAllStub.onCall(3).resolves([])
const req = {
query: {
@ -429,6 +429,8 @@ describe('LibraryController large-library browse contract', () => {
await LibraryController.getLibraryItems(req, res)
const response = res.json.firstCall.args[0]
expect(queryStub.calledTwice).to.equal(true)
expect(findAllStub.calledOnce).to.equal(true)
expect(response.total).to.equal(2)
expect(response.results).to.have.length(1)
expect(response.results[0].id).to.equal('li_3')
@ -451,43 +453,66 @@ describe('LibraryController large-library browse contract', () => {
bookAuthors: []
})
const findAllStub = sinon.stub(Database.bookModel, 'findAll')
sinon.stub(Database.bookModel, 'count').resolves(3)
findAllStub.onCall(0).resolves([
makeBook({ libraryItemId: 'li_1', filterSequence: '1', subseriesId: 'sub_1', subseriesName: 'Subseries 1' })
])
findAllStub.onCall(1).resolves([
const queryStub = sinon.stub(Database.sequelize, 'query')
queryStub.onCall(0).resolves([{ rawBookCount: 3, plainRowCount: 1, collapsedRowCount: 1 }])
queryStub.onCall(1).resolves([{ rowType: 'collapsed', anchorBookId: 'book_1', anchorLibraryItemId: 'li_1', subseriesId: 'sub_1' }])
const findAllStub = sinon.stub(Database.bookModel, 'findAll').resolves([
makeBook({ libraryItemId: 'li_1', filterSequence: '1', subseriesId: 'sub_1', subseriesName: 'Subseries 1' }),
makeBook({ libraryItemId: 'li_2', filterSequence: '2', subseriesId: 'sub_1', subseriesName: 'Subseries 1' })
])
findAllStub.onCall(2).resolves([
makeBook({ libraryItemId: 'li_3', filterSequence: '3' })
])
findAllStub.onCall(3).resolves([])
const req = {
query: {
filter: `series.${Buffer.from('series_1').toString('base64')}`,
collapseseries: '1',
limit: '1',
sort: 'sequence'
},
library: { id: 'lib_1', mediaType: 'book', isVirtual: false, settings: {} },
user: { id: 'user_1', checkCanAccessLibraryItem: () => true }
}
const res = { json: sinon.spy() }
const result = await libraryItemsBookFilters.getCollapsedSeriesWindow(
'lib_1',
'series_1',
{ id: 'user_1', canAccessExplicitContent: true, accessAllTags: true },
[],
{ sortBy: 'sequence', sortDesc: false, hideSingleBookSeries: false },
{ limit: 1, offset: 0 }
)
await LibraryController.getLibraryItems(req, res)
const response = res.json.firstCall.args[0]
expect(response.total).to.equal(2)
expect(response.results).to.have.length(1)
expect(response.results[0].collapsedSeries).to.include({
expect(queryStub.calledTwice).to.equal(true)
expect(findAllStub.calledOnce).to.equal(true)
expect(result.count).to.equal(2)
expect(result.libraryItems).to.have.length(1)
expect(result.libraryItems[0].collapsedSeries).to.include({
id: 'sub_1',
name: 'Subseries 1',
numBooks: 2
name: 'Subseries 1'
})
expect(response.results[0].collapsedSeries.libraryItemIds).to.deep.equal(['li_1', 'li_2'])
expect(response.results[0].collapsedSeries.seriesSequenceList).to.equal('1-2')
expect(result.libraryItems[0].collapsedSeries.books.map((book) => book.id)).to.deep.equal(['li_1', 'li_2'])
})
it('uses bounded anchor queries instead of scanning the full filtered series for each page', async () => {
const queryStub = sinon.stub(Database.sequelize, 'query')
queryStub.onCall(0).resolves([[{ rawBookCount: 1000, plainRowCount: 400, collapsedRowCount: 200 }]])
queryStub.onCall(1).resolves([[{ rowType: 'plain', anchorBookId: 'book_1', anchorLibraryItemId: 'li_1', subseriesId: null }]])
const findAllStub = sinon.stub(Database.bookModel, 'findAll').resolves([
{
id: 'book_1',
libraryItem: {
id: 'li_1',
mediaType: 'book',
toOldJSONMinified() {
return { id: 'li_1', media: { metadata: {} } }
}
},
series: [{ id: 'series_1', name: 'Main Series', bookSeries: { sequence: '1' } }],
bookAuthors: []
}
])
const result = await libraryItemsBookFilters.getCollapsedSeriesWindow(
'lib_1',
'series_1',
{ id: 'user_1', canAccessExplicitContent: true, accessAllTags: true },
[],
{ sortBy: 'sequence', sortDesc: false, hideSingleBookSeries: false },
{ limit: 20, offset: 40 }
)
expect(queryStub.calledTwice).to.equal(true)
expect(findAllStub.callCount).to.equal(1)
expect(result.count).to.equal(600)
})
it('adds a deterministic libraryItem.id tie-breaker for keyset title browse with duplicate titles', async () => {