From d39de74db0d4b0b6cd27ca2b8456c3e2bfaf252a Mon Sep 17 00:00:00 2001 From: Jonathan Finley Date: Fri, 13 Mar 2026 22:20:31 -0400 Subject: [PATCH] perf: add large-library browse indexes --- .../v2.32.6-large-library-browse-indexes.js | 132 ++++++++++++++++++ ...ge-library-browse-indexes.postgres.test.js | 95 +++++++++++++ ....32.6-large-library-browse-indexes.test.js | 106 ++++++++++++++ 3 files changed, 333 insertions(+) create mode 100644 server/migrations/v2.32.6-large-library-browse-indexes.js create mode 100644 test/server/migrations/v2.32.6-large-library-browse-indexes.postgres.test.js create mode 100644 test/server/migrations/v2.32.6-large-library-browse-indexes.test.js diff --git a/server/migrations/v2.32.6-large-library-browse-indexes.js b/server/migrations/v2.32.6-large-library-browse-indexes.js new file mode 100644 index 000000000..01b58be11 --- /dev/null +++ b/server/migrations/v2.32.6-large-library-browse-indexes.js @@ -0,0 +1,132 @@ +/** + * @typedef MigrationContext + * @property {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object. + * @property {import('../Logger')} logger - a Logger object. + * + * @typedef MigrationOptions + * @property {MigrationContext} context - an object containing the migration context. + */ + +const migrationVersion = '2.32.6' +const migrationName = `${migrationVersion}-large-library-browse-indexes` +const loggerPrefix = `[${migrationVersion} migration]` + +const indexesToCreate = [ + { + tableName: 'libraryItems', + indexName: 'library_items_library_media_type_title_id', + getFields(dialect) { + return ['libraryId', 'mediaType', dialect === 'sqlite' ? { name: 'title', collate: 'NOCASE' } : 'title', 'id'] + } + }, + { + tableName: 'libraryItems', + indexName: 'library_items_library_media_type_title_ignore_prefix_id', + getFields(dialect) { + return ['libraryId', 'mediaType', dialect === 'sqlite' ? { name: 'titleIgnorePrefix', collate: 'NOCASE' } : 'titleIgnorePrefix', 'id'] + } + }, + { + tableName: 'libraryItems', + indexName: 'library_items_library_media_type_created_at_id', + getFields() { + return ['libraryId', 'mediaType', 'createdAt', 'id'] + } + }, + { + tableName: 'mediaProgresses', + indexName: 'media_progress_user_updated_at_id', + getFields() { + return ['userId', 'updatedAt', 'id'] + } + } +] + +async function tableExists(queryInterface, dialect, tableName) { + try { + const query = + dialect === 'sqlite' + ? `SELECT name FROM sqlite_master WHERE type='table' AND name='${tableName}'` + : `SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_name = '${tableName}'` + const result = await queryInterface.sequelize.query(query, { type: queryInterface.sequelize.QueryTypes.SELECT }) + return result.length > 0 + } catch { + return false + } +} + +async function safeCreateIndex(queryInterface, logger, dialect, tableName, indexName, fields) { + try { + if (!(await tableExists(queryInterface, dialect, tableName))) { + logger.info(`${loggerPrefix} Table ${tableName} does not exist, skipping index ${indexName}`) + return + } + + await queryInterface.addIndex(tableName, fields, { name: indexName }) + logger.info(`${loggerPrefix} Created index ${indexName} on ${tableName}`) + } catch (error) { + if (error.message?.includes('already exists') || error.name === 'SequelizeUniqueConstraintError') { + logger.info(`${loggerPrefix} Index ${indexName} already exists, skipping`) + return + } + + if (error.message?.includes('no such table') || error.message?.includes('does not exist')) { + logger.info(`${loggerPrefix} Table ${tableName} does not exist, skipping index ${indexName}`) + return + } + + throw error + } +} + +async function safeRemoveIndex(queryInterface, logger, tableName, indexName) { + try { + await queryInterface.removeIndex(tableName, indexName) + logger.info(`${loggerPrefix} Removed index ${indexName} from ${tableName}`) + } catch { + logger.info(`${loggerPrefix} Index ${indexName} does not exist or could not be removed`) + } +} + +/** + * @param {MigrationOptions} options - an object containing the migration context. + * @returns {Promise} + */ +async function up({ context: { queryInterface } }) { + const logger = require('../Logger') + const dialect = queryInterface.sequelize.getDialect() + logger.info(`${loggerPrefix} Running migration for dialect: ${dialect}`) + + const booksExist = await tableExists(queryInterface, dialect, 'books') + if (!booksExist) { + logger.info(`${loggerPrefix} Core tables do not exist yet (fresh database), skipping migration`) + return + } + + for (const index of indexesToCreate) { + await safeCreateIndex(queryInterface, logger, dialect, index.tableName, index.indexName, index.getFields(dialect)) + } + + logger.info(`${loggerPrefix} Migration completed successfully`) +} + +/** + * @param {MigrationOptions} options - an object containing the migration context. + * @returns {Promise} + */ +async function down({ context: { queryInterface } }) { + const logger = require('../Logger') + logger.info(`${loggerPrefix} Running downgrade migration...`) + + for (const index of indexesToCreate) { + await safeRemoveIndex(queryInterface, logger, index.tableName, index.indexName) + } + + logger.info(`${loggerPrefix} Downgrade migration completed`) +} + +module.exports = { + up, + down, + name: migrationName +} diff --git a/test/server/migrations/v2.32.6-large-library-browse-indexes.postgres.test.js b/test/server/migrations/v2.32.6-large-library-browse-indexes.postgres.test.js new file mode 100644 index 000000000..37f7234d2 --- /dev/null +++ b/test/server/migrations/v2.32.6-large-library-browse-indexes.postgres.test.js @@ -0,0 +1,95 @@ +const chai = require('chai') +const sinon = require('sinon') +const { expect } = chai + +const { DataTypes, Sequelize } = require('sequelize') +const Logger = require('../../../server/Logger') + +const { up } = require('../../../server/migrations/v2.32.6-large-library-browse-indexes') + +const TEST_POSTGRES_URL = process.env.TEST_POSTGRES_URL + +describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => { + if (!TEST_POSTGRES_URL) { + it('skips because TEST_POSTGRES_URL is not set', function() { + this.skip() + }) + return + } + + let sequelize + let queryInterface + + beforeEach(async () => { + sequelize = new Sequelize(TEST_POSTGRES_URL, { + dialect: 'postgres', + logging: false + }) + queryInterface = sequelize.getQueryInterface() + sinon.stub(Logger, 'info') + + await queryInterface.dropTable('mediaProgresses').catch(() => {}) + await queryInterface.dropTable('libraryItems').catch(() => {}) + await queryInterface.dropTable('books').catch(() => {}) + + await queryInterface.createTable('books', { + id: { type: DataTypes.UUID, allowNull: false, primaryKey: true } + }) + + await queryInterface.createTable('libraryItems', { + id: { type: DataTypes.UUID, allowNull: false, primaryKey: true }, + libraryId: { type: DataTypes.UUID, allowNull: false }, + mediaType: { type: DataTypes.STRING, allowNull: false }, + title: { type: DataTypes.TEXT, allowNull: true }, + titleIgnorePrefix: { type: DataTypes.TEXT, allowNull: true }, + createdAt: { type: DataTypes.DATE, allowNull: true } + }) + + await queryInterface.createTable('mediaProgresses', { + id: { type: DataTypes.UUID, allowNull: false, primaryKey: true }, + userId: { type: DataTypes.UUID, allowNull: false }, + updatedAt: { type: DataTypes.DATE, allowNull: true } + }) + }) + + afterEach(async () => { + sinon.restore() + if (queryInterface) { + await queryInterface.dropTable('mediaProgresses').catch(() => {}) + await queryInterface.dropTable('libraryItems').catch(() => {}) + await queryInterface.dropTable('books').catch(() => {}) + } + if (sequelize) { + await sequelize.close() + } + }) + + it('creates browse indexes with the expected names and column order', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const [rows] = await queryInterface.sequelize.query(` + SELECT indexname, indexdef + FROM pg_indexes + WHERE schemaname = 'public' + AND indexname IN ( + 'library_items_library_media_type_title_id', + 'library_items_library_media_type_title_ignore_prefix_id', + 'library_items_library_media_type_created_at_id', + 'media_progress_user_updated_at_id' + ) + ORDER BY indexname ASC + `) + + expect(rows.map((row) => row.indexname)).to.deep.equal([ + 'library_items_library_media_type_created_at_id', + 'library_items_library_media_type_title_id', + 'library_items_library_media_type_title_ignore_prefix_id', + 'media_progress_user_updated_at_id' + ]) + + expect(rows.find((row) => row.indexname === 'library_items_library_media_type_title_id').indexdef).to.include('("libraryId", "mediaType", title, id)') + expect(rows.find((row) => row.indexname === 'library_items_library_media_type_title_ignore_prefix_id').indexdef).to.include('("libraryId", "mediaType", "titleIgnorePrefix", id)') + expect(rows.find((row) => row.indexname === 'library_items_library_media_type_created_at_id').indexdef).to.include('("libraryId", "mediaType", "createdAt", id)') + expect(rows.find((row) => row.indexname === 'media_progress_user_updated_at_id').indexdef).to.include('("userId", "updatedAt", id)') + }) +}) diff --git a/test/server/migrations/v2.32.6-large-library-browse-indexes.test.js b/test/server/migrations/v2.32.6-large-library-browse-indexes.test.js new file mode 100644 index 000000000..64b54a41e --- /dev/null +++ b/test/server/migrations/v2.32.6-large-library-browse-indexes.test.js @@ -0,0 +1,106 @@ +const chai = require('chai') +const sinon = require('sinon') +const { expect } = chai + +const { DataTypes, Sequelize } = require('sequelize') +const Logger = require('../../../server/Logger') + +const { up } = require('../../../server/migrations/v2.32.6-large-library-browse-indexes') + +const normalizeWhitespace = (str) => str.replace(/\s+/g, ' ').trim().replace(/`/g, '') + +describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => { + let sequelize + let queryInterface + + beforeEach(async () => { + sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + queryInterface = sequelize.getQueryInterface() + sinon.stub(Logger, 'info') + + await queryInterface.createTable('books', { + id: { type: DataTypes.UUID, allowNull: false, primaryKey: true } + }) + + await queryInterface.createTable('libraryItems', { + id: { type: DataTypes.UUID, allowNull: false, primaryKey: true }, + libraryId: { type: DataTypes.UUID, allowNull: false }, + mediaType: { type: DataTypes.STRING, allowNull: false }, + title: { type: DataTypes.TEXT, allowNull: true }, + titleIgnorePrefix: { type: DataTypes.TEXT, allowNull: true }, + createdAt: { type: DataTypes.DATE, allowNull: true } + }) + + await queryInterface.createTable('mediaProgresses', { + id: { type: DataTypes.UUID, allowNull: false, primaryKey: true }, + userId: { type: DataTypes.UUID, allowNull: false }, + updatedAt: { type: DataTypes.DATE, allowNull: true } + }) + }) + + afterEach(async () => { + sinon.restore() + await sequelize.close() + }) + + it('creates browse indexes for title and progress keyset traversal', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const [[{ count: titleExactIndexCount }]] = await queryInterface.sequelize.query( + "SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_title_id'" + ) + expect(titleExactIndexCount).to.equal(1) + + const [[{ sql: titleExactIndexSql }]] = await queryInterface.sequelize.query( + "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_title_id'" + ) + expect(normalizeWhitespace(titleExactIndexSql)).to.equal( + normalizeWhitespace('CREATE INDEX library_items_library_media_type_title_id ON libraryItems (libraryId, mediaType, title COLLATE NOCASE, id)') + ) + + const [[{ count: titleIndexCount }]] = await queryInterface.sequelize.query( + "SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_title_ignore_prefix_id'" + ) + expect(titleIndexCount).to.equal(1) + + const [[{ sql: titleIndexSql }]] = await queryInterface.sequelize.query( + "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_title_ignore_prefix_id'" + ) + expect(normalizeWhitespace(titleIndexSql)).to.equal( + normalizeWhitespace('CREATE INDEX library_items_library_media_type_title_ignore_prefix_id ON libraryItems (libraryId, mediaType, titleIgnorePrefix COLLATE NOCASE, id)') + ) + + const [[{ sql: createdAtIndexSql }]] = await queryInterface.sequelize.query( + "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_created_at_id'" + ) + expect(normalizeWhitespace(createdAtIndexSql)).to.equal( + normalizeWhitespace('CREATE INDEX library_items_library_media_type_created_at_id ON libraryItems (libraryId, mediaType, createdAt, id)') + ) + + const [[{ sql: progressIndexSql }]] = await queryInterface.sequelize.query( + "SELECT sql FROM sqlite_master WHERE type='index' AND name='media_progress_user_updated_at_id'" + ) + expect(normalizeWhitespace(progressIndexSql)).to.equal( + normalizeWhitespace('CREATE INDEX media_progress_user_updated_at_id ON mediaProgresses (userId, updatedAt, id)') + ) + }) + + it('is idempotent', async () => { + await up({ context: { queryInterface, logger: Logger } }) + await up({ context: { queryInterface, logger: Logger } }) + + const indexes = [ + 'library_items_library_media_type_title_id', + 'library_items_library_media_type_title_ignore_prefix_id', + 'library_items_library_media_type_created_at_id', + 'media_progress_user_updated_at_id' + ] + + for (const indexName of indexes) { + const [[{ count }]] = await queryInterface.sequelize.query( + `SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='${indexName}'` + ) + expect(count, indexName).to.equal(1) + } + }) +})