diff --git a/server/managers/ApiCacheManager.js b/server/managers/ApiCacheManager.js index 2d8eece85..8a67e4898 100644 --- a/server/managers/ApiCacheManager.js +++ b/server/managers/ApiCacheManager.js @@ -5,6 +5,9 @@ const Database = require('../Database') class ApiCacheManager { defaultCacheOptions = { max: 1000, maxSize: 10 * 1000 * 1000, sizeCalculation: (item) => item.body.length + JSON.stringify(item.headers).length } defaultTtlOptions = { ttl: 30 * 60 * 1000 } + highChurnModels = new Set(['session', 'mediaProgress', 'playbackSession', 'device']) + modelsInvalidatingPersonalized = new Set(['mediaProgress']) + modelsInvalidatingMe = new Set(['session', 'mediaProgress', 'playbackSession', 'device']) constructor(cache = new LRUCache(this.defaultCacheOptions), ttlOptions = this.defaultTtlOptions) { this.cache = cache @@ -16,8 +19,44 @@ class ApiCacheManager { hooks.forEach((hook) => database.sequelize.addHook(hook, (model) => this.clear(model, hook))) } + getModelName(model) { + if (typeof model?.name === 'string') return model.name + if (typeof model?.model?.name === 'string') return model.model.name + if (typeof model?.constructor?.name === 'string' && model.constructor.name !== 'Object') return model.constructor.name + return 'unknown' + } + + clearByUrlPattern(urlPattern) { + let removed = 0 + for (const key of this.cache.keys()) { + try { + const parsed = JSON.parse(key) + if (typeof parsed?.url === 'string' && urlPattern.test(parsed.url)) { + if (this.cache.delete(key)) removed++ + } + } catch { + if (this.cache.delete(key)) removed++ + } + } + return removed + } + + clearUserProgressSlices(modelName, hook) { + const removedPersonalized = this.modelsInvalidatingPersonalized.has(modelName) ? this.clearByUrlPattern(/^\/libraries\/[^/]+\/personalized/) : 0 + const removedMe = this.modelsInvalidatingMe.has(modelName) ? this.clearByUrlPattern(/^\/me(\/|\?|$)/) : 0 + Logger.debug( + `[ApiCacheManager] ${modelName}.${hook}: cleared user-progress cache slices (personalized=${removedPersonalized}, me=${removedMe})` + ) + } + clear(model, hook) { - Logger.debug(`[ApiCacheManager] ${model.constructor.name}.${hook}: Clearing cache`) + const modelName = this.getModelName(model) + if (this.highChurnModels.has(modelName)) { + this.clearUserProgressSlices(modelName, hook) + return + } + + Logger.debug(`[ApiCacheManager] ${modelName}.${hook}: Clearing cache`) this.cache.clear() } diff --git a/server/migrations/v2.32.2-add-discover-query-indexes.js b/server/migrations/v2.32.2-add-discover-query-indexes.js new file mode 100644 index 000000000..26b12383d --- /dev/null +++ b/server/migrations/v2.32.2-add-discover-query-indexes.js @@ -0,0 +1,74 @@ +/** + * @typedef MigrationContext + * @property {import('sequelize').QueryInterface} queryInterface + * @property {import('../Logger')} logger + * + * @typedef MigrationOptions + * @property {MigrationContext} context + */ + +const migrationVersion = '2.32.2' +const migrationName = `${migrationVersion}-add-discover-query-indexes` +const loggerPrefix = `[${migrationVersion} migration]` + +const indexes = [ + { + table: 'mediaProgresses', + name: 'media_progress_user_item_finished_time', + fields: ['userId', 'mediaItemId', 'isFinished', 'currentTime'] + }, + { + table: 'bookSeries', + name: 'book_series_series_book', + fields: ['seriesId', 'bookId'] + } +] + +async function up({ context: { queryInterface, logger } }) { + logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) + + for (const index of indexes) { + await addIndexIfMissing(queryInterface, logger, index) + } + + logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`) +} + +async function down({ context: { queryInterface, logger } }) { + logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) + + for (const index of indexes) { + await removeIndexIfExists(queryInterface, logger, index) + } + + logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`) +} + +async function addIndexIfMissing(queryInterface, logger, index) { + const existing = await queryInterface.showIndex(index.table) + if (existing.some((i) => i.name === index.name)) { + logger.info(`${loggerPrefix} index ${index.name} already exists on ${index.table}`) + return + } + + logger.info(`${loggerPrefix} adding index ${index.name} on ${index.table}(${index.fields.join(', ')})`) + await queryInterface.addIndex(index.table, { + name: index.name, + fields: index.fields + }) + logger.info(`${loggerPrefix} added index ${index.name}`) +} + +async function removeIndexIfExists(queryInterface, logger, index) { + const existing = await queryInterface.showIndex(index.table) + if (!existing.some((i) => i.name === index.name)) { + logger.info(`${loggerPrefix} index ${index.name} does not exist on ${index.table}`) + return + } + + logger.info(`${loggerPrefix} removing index ${index.name}`) + await queryInterface.removeIndex(index.table, index.name) + logger.info(`${loggerPrefix} removed index ${index.name}`) +} + +module.exports = { up, down } diff --git a/server/models/LibraryItem.js b/server/models/LibraryItem.js index 16a521615..b33fa585c 100644 --- a/server/models/LibraryItem.js +++ b/server/models/LibraryItem.js @@ -340,6 +340,15 @@ class LibraryItem extends Model { const shelves = [] + const timed = async (loader) => { + const start = Date.now() + const payload = await loader() + return { + payload, + elapsedSeconds: ((Date.now() - start) / 1000).toFixed(2) + } + } + // "Continue Listening" shelf const itemsInProgressPayload = await libraryFilters.getMediaItemsInProgress(library, user, include, limit, false) if (itemsInProgressPayload.items.length) { @@ -371,11 +380,18 @@ class LibraryItem extends Model { } Logger.debug(`Loaded ${itemsInProgressPayload.items.length} of ${itemsInProgressPayload.count} items for "Continue Listening/Reading" in ${((Date.now() - fullStart) / 1000).toFixed(2)}s`) - let start = Date.now() if (library.isBook) { - start = Date.now() + const [continueSeriesResult, mostRecentResult, seriesMostRecentResult, discoverResult, mediaFinishedResult, newestAuthorsResult] = await Promise.all([ + timed(() => libraryFilters.getLibraryItemsContinueSeries(library, user, include, limit)), + timed(() => libraryFilters.getLibraryItemsMostRecentlyAdded(library, user, include, limit)), + timed(() => libraryFilters.getSeriesMostRecentlyAdded(library, user, include, 5)), + timed(() => libraryFilters.getLibraryItemsToDiscover(library, user, include, limit)), + timed(() => libraryFilters.getMediaFinished(library, user, include, limit)), + timed(() => libraryFilters.getNewestAuthors(library, user, limit)) + ]) + + const continueSeriesPayload = continueSeriesResult.payload // "Continue Series" shelf - const continueSeriesPayload = await libraryFilters.getLibraryItemsContinueSeries(library, user, include, limit) if (continueSeriesPayload.libraryItems.length) { shelves.push({ id: 'continue-series', @@ -386,42 +402,24 @@ class LibraryItem extends Model { total: continueSeriesPayload.count }) } - Logger.debug(`Loaded ${continueSeriesPayload.libraryItems.length} of ${continueSeriesPayload.count} items for "Continue Series" in ${((Date.now() - start) / 1000).toFixed(2)}s`) - } else if (library.isPodcast) { - // "Newest Episodes" shelf - const newestEpisodesPayload = await libraryFilters.getNewestPodcastEpisodes(library, user, limit) - if (newestEpisodesPayload.libraryItems.length) { + Logger.debug(`Loaded ${continueSeriesPayload.libraryItems.length} of ${continueSeriesPayload.count} items for "Continue Series" in ${continueSeriesResult.elapsedSeconds}s`) + + const mostRecentPayload = mostRecentResult.payload + // "Recently Added" shelf + if (mostRecentPayload.libraryItems.length) { shelves.push({ - id: 'newest-episodes', - label: 'Newest Episodes', - labelStringKey: 'LabelNewestEpisodes', - type: 'episode', - entities: newestEpisodesPayload.libraryItems, - total: newestEpisodesPayload.count + id: 'recently-added', + label: 'Recently Added', + labelStringKey: 'LabelRecentlyAdded', + type: library.mediaType, + entities: mostRecentPayload.libraryItems, + total: mostRecentPayload.count }) } - Logger.debug(`Loaded ${newestEpisodesPayload.libraryItems.length} of ${newestEpisodesPayload.count} episodes for "Newest Episodes" in ${((Date.now() - start) / 1000).toFixed(2)}s`) - } + Logger.debug(`Loaded ${mostRecentPayload.libraryItems.length} of ${mostRecentPayload.count} items for "Recently Added" in ${mostRecentResult.elapsedSeconds}s`) - start = Date.now() - // "Recently Added" shelf - const mostRecentPayload = await libraryFilters.getLibraryItemsMostRecentlyAdded(library, user, include, limit) - if (mostRecentPayload.libraryItems.length) { - shelves.push({ - id: 'recently-added', - label: 'Recently Added', - labelStringKey: 'LabelRecentlyAdded', - type: library.mediaType, - entities: mostRecentPayload.libraryItems, - total: mostRecentPayload.count - }) - } - Logger.debug(`Loaded ${mostRecentPayload.libraryItems.length} of ${mostRecentPayload.count} items for "Recently Added" in ${((Date.now() - start) / 1000).toFixed(2)}s`) - - if (library.isBook) { - start = Date.now() + const seriesMostRecentPayload = seriesMostRecentResult.payload // "Recent Series" shelf - const seriesMostRecentPayload = await libraryFilters.getSeriesMostRecentlyAdded(library, user, include, 5) if (seriesMostRecentPayload.series.length) { shelves.push({ id: 'recent-series', @@ -432,11 +430,10 @@ class LibraryItem extends Model { total: seriesMostRecentPayload.count }) } - Logger.debug(`Loaded ${seriesMostRecentPayload.series.length} of ${seriesMostRecentPayload.count} series for "Recent Series" in ${((Date.now() - start) / 1000).toFixed(2)}s`) + Logger.debug(`Loaded ${seriesMostRecentPayload.series.length} of ${seriesMostRecentPayload.count} series for "Recent Series" in ${seriesMostRecentResult.elapsedSeconds}s`) - start = Date.now() + const discoverLibraryItemsPayload = discoverResult.payload // "Discover" shelf - const discoverLibraryItemsPayload = await libraryFilters.getLibraryItemsToDiscover(library, user, include, limit) if (discoverLibraryItemsPayload.libraryItems.length) { shelves.push({ id: 'discover', @@ -447,45 +444,41 @@ class LibraryItem extends Model { total: discoverLibraryItemsPayload.count }) } - Logger.debug(`Loaded ${discoverLibraryItemsPayload.libraryItems.length} of ${discoverLibraryItemsPayload.count} items for "Discover" in ${((Date.now() - start) / 1000).toFixed(2)}s`) - } + Logger.debug(`Loaded ${discoverLibraryItemsPayload.libraryItems.length} of ${discoverLibraryItemsPayload.count} items for "Discover" in ${discoverResult.elapsedSeconds}s`) - start = Date.now() - // "Listen Again" shelf - const mediaFinishedPayload = await libraryFilters.getMediaFinished(library, user, include, limit) - if (mediaFinishedPayload.items.length) { - const ebookOnlyItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.ebookFormat && !li.media.numTracks) - const audioItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.numTracks || li.mediaType === 'podcast') + const mediaFinishedPayload = mediaFinishedResult.payload + // "Listen Again" shelf + if (mediaFinishedPayload.items.length) { + const ebookOnlyItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.ebookFormat && !li.media.numTracks) + const audioItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.numTracks || li.mediaType === 'podcast') - if (audioItemsInProgress.length) { - shelves.push({ - id: 'listen-again', - label: 'Listen Again', - labelStringKey: 'LabelListenAgain', - type: library.isPodcast ? 'episode' : 'book', - entities: audioItemsInProgress, - total: mediaFinishedPayload.count - }) + if (audioItemsInProgress.length) { + shelves.push({ + id: 'listen-again', + label: 'Listen Again', + labelStringKey: 'LabelListenAgain', + type: library.isPodcast ? 'episode' : 'book', + entities: audioItemsInProgress, + total: mediaFinishedPayload.count + }) + } + + if (ebookOnlyItemsInProgress.length) { + // "Read Again" shelf + shelves.push({ + id: 'read-again', + label: 'Read Again', + labelStringKey: 'LabelReadAgain', + type: 'book', + entities: ebookOnlyItemsInProgress, + total: mediaFinishedPayload.count + }) + } } + Logger.debug(`Loaded ${mediaFinishedPayload.items.length} of ${mediaFinishedPayload.count} items for "Listen/Read Again" in ${mediaFinishedResult.elapsedSeconds}s`) - // "Read Again" shelf - if (ebookOnlyItemsInProgress.length) { - shelves.push({ - id: 'read-again', - label: 'Read Again', - labelStringKey: 'LabelReadAgain', - type: 'book', - entities: ebookOnlyItemsInProgress, - total: mediaFinishedPayload.count - }) - } - } - Logger.debug(`Loaded ${mediaFinishedPayload.items.length} of ${mediaFinishedPayload.count} items for "Listen/Read Again" in ${((Date.now() - start) / 1000).toFixed(2)}s`) - - if (library.isBook) { - start = Date.now() + const newestAuthorsPayload = newestAuthorsResult.payload // "Newest Authors" shelf - const newestAuthorsPayload = await libraryFilters.getNewestAuthors(library, user, limit) if (newestAuthorsPayload.authors.length) { shelves.push({ id: 'newest-authors', @@ -496,7 +489,72 @@ class LibraryItem extends Model { total: newestAuthorsPayload.count }) } - Logger.debug(`Loaded ${newestAuthorsPayload.authors.length} of ${newestAuthorsPayload.count} authors for "Newest Authors" in ${((Date.now() - start) / 1000).toFixed(2)}s`) + Logger.debug(`Loaded ${newestAuthorsPayload.authors.length} of ${newestAuthorsPayload.count} authors for "Newest Authors" in ${newestAuthorsResult.elapsedSeconds}s`) + } else if (library.isPodcast) { + const [newestEpisodesResult, mostRecentResult, mediaFinishedResult] = await Promise.all([ + timed(() => libraryFilters.getNewestPodcastEpisodes(library, user, limit)), + timed(() => libraryFilters.getLibraryItemsMostRecentlyAdded(library, user, include, limit)), + timed(() => libraryFilters.getMediaFinished(library, user, include, limit)) + ]) + + const newestEpisodesPayload = newestEpisodesResult.payload + // "Newest Episodes" shelf + if (newestEpisodesPayload.libraryItems.length) { + shelves.push({ + id: 'newest-episodes', + label: 'Newest Episodes', + labelStringKey: 'LabelNewestEpisodes', + type: 'episode', + entities: newestEpisodesPayload.libraryItems, + total: newestEpisodesPayload.count + }) + } + Logger.debug(`Loaded ${newestEpisodesPayload.libraryItems.length} of ${newestEpisodesPayload.count} episodes for "Newest Episodes" in ${newestEpisodesResult.elapsedSeconds}s`) + + const mostRecentPayload = mostRecentResult.payload + // "Recently Added" shelf + if (mostRecentPayload.libraryItems.length) { + shelves.push({ + id: 'recently-added', + label: 'Recently Added', + labelStringKey: 'LabelRecentlyAdded', + type: library.mediaType, + entities: mostRecentPayload.libraryItems, + total: mostRecentPayload.count + }) + } + Logger.debug(`Loaded ${mostRecentPayload.libraryItems.length} of ${mostRecentPayload.count} items for "Recently Added" in ${mostRecentResult.elapsedSeconds}s`) + + const mediaFinishedPayload = mediaFinishedResult.payload + // "Listen Again" shelf + if (mediaFinishedPayload.items.length) { + const ebookOnlyItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.ebookFormat && !li.media.numTracks) + const audioItemsInProgress = mediaFinishedPayload.items.filter((li) => li.media.numTracks || li.mediaType === 'podcast') + + if (audioItemsInProgress.length) { + shelves.push({ + id: 'listen-again', + label: 'Listen Again', + labelStringKey: 'LabelListenAgain', + type: 'episode', + entities: audioItemsInProgress, + total: mediaFinishedPayload.count + }) + } + + if (ebookOnlyItemsInProgress.length) { + // "Read Again" shelf + shelves.push({ + id: 'read-again', + label: 'Read Again', + labelStringKey: 'LabelReadAgain', + type: 'book', + entities: ebookOnlyItemsInProgress, + total: mediaFinishedPayload.count + }) + } + } + Logger.debug(`Loaded ${mediaFinishedPayload.items.length} of ${mediaFinishedPayload.count} items for "Listen/Read Again" in ${mediaFinishedResult.elapsedSeconds}s`) } Logger.debug(`Loaded ${shelves.length} personalized shelves in ${((Date.now() - fullStart) / 1000).toFixed(2)}s`) diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index 7ae1dc866..fbe0c4f0d 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -888,28 +888,79 @@ module.exports = { }) } - // Step 2: Get books not started and not in a series OR is the first book of a series not started (ordered randomly) - const { rows: books, count } = await Database.bookModel.findAndCountAll({ - where: [ - { - '$mediaProgresses.isFinished$': { - [Sequelize.Op.or]: [null, 0] - }, - '$mediaProgresses.currentTime$': { - [Sequelize.Op.or]: [null, 0] - }, - [Sequelize.Op.or]: [ - Sequelize.where(Sequelize.literal(`(SELECT COUNT(*) FROM bookSeries bs where bs.bookId = book.id)`), 0), - { - id: { - [Sequelize.Op.in]: booksFromSeriesToInclude - } - } - ] + const discoverWhere = [ + { + '$mediaProgresses.isFinished$': { + [Sequelize.Op.or]: [null, 0] }, - ...userPermissionBookWhere.bookWhere - ], + '$mediaProgresses.currentTime$': { + [Sequelize.Op.or]: [null, 0] + }, + [Sequelize.Op.or]: [ + Sequelize.where(Sequelize.literal(`(SELECT COUNT(*) FROM bookSeries bs where bs.bookId = book.id)`), 0), + { + id: { + [Sequelize.Op.in]: booksFromSeriesToInclude + } + } + ] + }, + ...userPermissionBookWhere.bookWhere + ] + + const baseDiscoverInclude = [ + { + model: Database.libraryItemModel, + where: { + libraryId + } + }, + { + model: Database.mediaProgressModel, + where: { + userId: user.id + }, + required: false + } + ] + + // Step 2a: Count with lightweight includes only + const count = await Database.bookModel.count({ + where: discoverWhere, replacements: userPermissionBookWhere.replacements, + include: baseDiscoverInclude, + distinct: true, + col: 'id', + subQuery: false + }) + + // Step 2b: Select random IDs with lightweight includes only + const randomSelection = await Database.bookModel.findAll({ + attributes: ['id'], + where: discoverWhere, + replacements: userPermissionBookWhere.replacements, + include: baseDiscoverInclude, + subQuery: false, + distinct: true, + limit, + order: Database.sequelize.random() + }) + + const selectedIds = randomSelection.map((b) => b.id).filter(Boolean) + if (!selectedIds.length) { + return { + libraryItems: [], + count + } + } + + // Step 2c: Hydrate selected IDs with full metadata for API response + const books = await Database.bookModel.findAll({ + where: { + id: { + [Sequelize.Op.in]: selectedIds + } + }, include: [ { model: Database.libraryItemModel, @@ -918,13 +969,6 @@ module.exports = { }, include: libraryItemIncludes }, - { - model: Database.mediaProgressModel, - where: { - userId: user.id - }, - required: false - }, { model: Database.bookAuthorModel, attributes: ['authorId'], @@ -942,14 +986,14 @@ module.exports = { separate: true } ], - subQuery: false, - distinct: true, - limit, - order: Database.sequelize.random() + subQuery: false }) + const booksById = new Map(books.map((b) => [b.id, b])) + const orderedBooks = selectedIds.map((id) => booksById.get(id)).filter(Boolean) + // Step 3: Map books to library items - const libraryItems = books.map((bookExpanded) => { + const libraryItems = orderedBooks.map((bookExpanded) => { const libraryItem = bookExpanded.libraryItem const book = bookExpanded delete book.libraryItem