diff --git a/server/Database.js b/server/Database.js index c20b78785..875f34047 100644 --- a/server/Database.js +++ b/server/Database.js @@ -593,7 +593,6 @@ class Database { return cacheEntry } - delete this.libraryFilterData[libraryId] return undefined } @@ -609,6 +608,15 @@ class Database { return cacheEntry } + refreshLibraryFilterCache(libraryId, ttlMs = this.libraryFilterDataTtlMs) { + const cacheEntry = this.libraryFilterData[libraryId] + if (!cacheEntry) return undefined + + cacheEntry.loadedAt = Date.now() + cacheEntry.expiresAt = cacheEntry.loadedAt + ttlMs + return cacheEntry + } + invalidateLibraryFilterCache(libraryId) { delete this.libraryFilterData[libraryId] } diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index 5dbb3787b..7aa906479 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -450,11 +450,13 @@ module.exports = { * @returns {Promise} */ async getFilterData(mediaType, libraryId) { + const cachedFilterData = Database.getLibraryFilterCache(libraryId, { includeExpired: true }) const freshFilterData = Database.getLibraryFilterCache(libraryId) if (freshFilterData) { return freshFilterData } const start = Date.now() // Temp for checking load times + const lastLoadedAt = cachedFilterData?.loadedAt || 0 const data = { authors: [], @@ -494,6 +496,30 @@ module.exports = { } }) + const changedPodcasts = cachedFilterData ? await podcastModelCount({ + include: { + model: Database.libraryItemModel, + attributes: [], + where: { + libraryId: libraryId, + updatedAt: { + [Sequelize.Op.gt]: new Date(lastLoadedAt) + } + } + }, + where: { + updatedAt: { + [Sequelize.Op.gt]: new Date(lastLoadedAt) + } + }, + limit: 1 + }) : 1 + + if (cachedFilterData && changedPodcasts === 0 && podcastCountFromDatabase === cachedFilterData.podcastCount) { + Logger.debug(`Filter data for ${libraryId} has not changed, returning cached data and updating cache time after ${((Date.now() - start) / 1000).toFixed(2)}s`) + return Database.refreshLibraryFilterCache(libraryId) + } + const findAll = process.env.QUERY_PROFILING ? profile(Database.podcastModel.findAll.bind(Database.podcastModel)) : Database.podcastModel.findAll.bind(Database.podcastModel) const podcasts = await findAll({ include: { @@ -520,14 +546,42 @@ module.exports = { // Set podcast count for later comparison data.podcastCount = podcastCountFromDatabase } else { - const [bookCountFromDatabase, seriesCountFromDatabase, authorCountFromDatabase] = await Promise.all([ + const [bookCountFromDatabase, seriesCountFromDatabase, authorCountFromDatabase, changedBooks, changedSeries, changedAuthors] = await Promise.all([ Database.bookModel.count({ include: { model: Database.libraryItemModel, attributes: [], where: { libraryId } } }), Database.seriesModel.count({ where: { libraryId } }), - Database.authorModel.count({ where: { libraryId } }) + Database.authorModel.count({ where: { libraryId } }), + cachedFilterData ? Database.bookModel.count({ + include: { + model: Database.libraryItemModel, + attributes: [], + where: { libraryId, updatedAt: { [Sequelize.Op.gt]: new Date(lastLoadedAt) } } + }, + where: { updatedAt: { [Sequelize.Op.gt]: new Date(lastLoadedAt) } }, + limit: 1 + }) : Promise.resolve(1), + cachedFilterData ? Database.seriesModel.count({ + where: { libraryId, updatedAt: { [Sequelize.Op.gt]: new Date(lastLoadedAt) } }, + limit: 1 + }) : Promise.resolve(1), + cachedFilterData ? Database.authorModel.count({ + where: { libraryId, updatedAt: { [Sequelize.Op.gt]: new Date(lastLoadedAt) } }, + limit: 1 + }) : Promise.resolve(1) ]) + if ( + cachedFilterData && + changedBooks + changedSeries + changedAuthors === 0 && + bookCountFromDatabase === cachedFilterData.bookCount && + seriesCountFromDatabase === cachedFilterData.seriesCount && + authorCountFromDatabase === cachedFilterData.authorCount + ) { + Logger.debug(`Filter data for ${libraryId} has not changed, returning cached data and updating cache time after ${((Date.now() - start) / 1000).toFixed(2)}s`) + return Database.refreshLibraryFilterCache(libraryId) + } + // Store the counts for later comparison data.bookCount = bookCountFromDatabase data.seriesCount = seriesCountFromDatabase diff --git a/test/server/controllers/LibraryController.largeLibraryBrowse.test.js b/test/server/controllers/LibraryController.largeLibraryBrowse.test.js index af0a9b900..2d68150ff 100644 --- a/test/server/controllers/LibraryController.largeLibraryBrowse.test.js +++ b/test/server/controllers/LibraryController.largeLibraryBrowse.test.js @@ -753,6 +753,56 @@ describe('LibraryController large-library browse contract', () => { expect(Database.bookModel.count.called).to.equal(true) }) + it('refreshes an expired cache entry without rebuilding when cheap change checks show no changes', async () => { + const clock = sinon.useFakeTimers({ now: 10_000 }) + Database.libraryFilterData.lib_1 = { + authors: [{ id: 'au_1', name: 'Author 1' }], + genres: [], + tags: [], + series: [{ id: 'se_1', name: 'Series 1' }], + narrators: [], + languages: [], + publishers: [], + publishedDecades: [], + bookCount: 3, + seriesCount: 1, + authorCount: 1, + podcastCount: 0, + ebookCount: 0, + numIssues: 0, + loadedAt: 1_000, + expiresAt: 2_000 + } + + sinon.stub(Database.bookModel, 'count') + .onFirstCall().resolves(3) + .onSecondCall().resolves(0) + sinon.stub(Database.seriesModel, 'count') + .onFirstCall().resolves(1) + .onSecondCall().resolves(0) + sinon.stub(Database.authorModel, 'count') + .onFirstCall().resolves(1) + .onSecondCall().resolves(0) + const genresFindAllStub = sinon.stub(Database.genreModel, 'findAll').resolves([]) + const tagsFindAllStub = sinon.stub(Database.tagModel, 'findAll').resolves([]) + const narratorsFindAllStub = sinon.stub(Database.narratorModel, 'findAll').resolves([]) + const booksFindAllStub = sinon.stub(Database.bookModel, 'findAll').resolves([]) + const seriesFindAllStub = sinon.stub(Database.seriesModel, 'findAll').resolves([]) + const authorsFindAllStub = sinon.stub(Database.authorModel, 'findAll').resolves([]) + + const data = await libraryFilters.getFilterData('book', 'lib_1') + + expect(data.authors).to.deep.equal([{ id: 'au_1', name: 'Author 1' }]) + expect(genresFindAllStub.called).to.equal(false) + expect(tagsFindAllStub.called).to.equal(false) + expect(narratorsFindAllStub.called).to.equal(false) + expect(booksFindAllStub.called).to.equal(false) + expect(seriesFindAllStub.called).to.equal(false) + expect(authorsFindAllStub.called).to.equal(false) + expect(Database.libraryFilterData.lib_1.loadedAt).to.equal(clock.now) + expect(Database.libraryFilterData.lib_1.expiresAt).to.equal(clock.now + Database.libraryFilterDataTtlMs) + }) + it('invalidates filter cache when a library write path changes library content', async () => { Database.setLibraryFilterCache('lib_1', { authors: [] }, 60_000) sinon.stub(Database.libraryItemModel, 'findAll').resolves([