perf: add large-library browse indexes

server/migrations/v2.32.6-large-library-browse-indexes.js

@@ -0,0 +1,132 @@
+/**
+ * @typedef MigrationContext
+ * @property {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
+ * @property {import('../Logger')} logger - a Logger object.
+ *
+ * @typedef MigrationOptions
+ * @property {MigrationContext} context - an object containing the migration context.
+ */
+
+const migrationVersion = '2.32.6'
+const migrationName = `${migrationVersion}-large-library-browse-indexes`
+const loggerPrefix = `[${migrationVersion} migration]`
+
+const indexesToCreate = [
+  {
+    tableName: 'libraryItems',
+    indexName: 'library_items_library_media_type_title_id',
+    getFields(dialect) {
+      return ['libraryId', 'mediaType', dialect === 'sqlite' ? { name: 'title', collate: 'NOCASE' } : 'title', 'id']
+    }
+  },
+  {
+    tableName: 'libraryItems',
+    indexName: 'library_items_library_media_type_title_ignore_prefix_id',
+    getFields(dialect) {
+      return ['libraryId', 'mediaType', dialect === 'sqlite' ? { name: 'titleIgnorePrefix', collate: 'NOCASE' } : 'titleIgnorePrefix', 'id']
+    }
+  },
+  {
+    tableName: 'libraryItems',
+    indexName: 'library_items_library_media_type_created_at_id',
+    getFields() {
+      return ['libraryId', 'mediaType', 'createdAt', 'id']
+    }
+  },
+  {
+    tableName: 'mediaProgresses',
+    indexName: 'media_progress_user_updated_at_id',
+    getFields() {
+      return ['userId', 'updatedAt', 'id']
+    }
+  }
+]
+
+async function tableExists(queryInterface, dialect, tableName) {
+  try {
+    const query =
+      dialect === 'sqlite'
+        ? `SELECT name FROM sqlite_master WHERE type='table' AND name='${tableName}'`
+        : `SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_name = '${tableName}'`
+    const result = await queryInterface.sequelize.query(query, { type: queryInterface.sequelize.QueryTypes.SELECT })
+    return result.length > 0
+  } catch {
+    return false
+  }
+}
+
+async function safeCreateIndex(queryInterface, logger, dialect, tableName, indexName, fields) {
+  try {
+    if (!(await tableExists(queryInterface, dialect, tableName))) {
+      logger.info(`${loggerPrefix} Table ${tableName} does not exist, skipping index ${indexName}`)
+      return
+    }
+
+    await queryInterface.addIndex(tableName, fields, { name: indexName })
+    logger.info(`${loggerPrefix} Created index ${indexName} on ${tableName}`)
+  } catch (error) {
+    if (error.message?.includes('already exists') || error.name === 'SequelizeUniqueConstraintError') {
+      logger.info(`${loggerPrefix} Index ${indexName} already exists, skipping`)
+      return
+    }
+
+    if (error.message?.includes('no such table') || error.message?.includes('does not exist')) {
+      logger.info(`${loggerPrefix} Table ${tableName} does not exist, skipping index ${indexName}`)
+      return
+    }
+
+    throw error
+  }
+}
+
+async function safeRemoveIndex(queryInterface, logger, tableName, indexName) {
+  try {
+    await queryInterface.removeIndex(tableName, indexName)
+    logger.info(`${loggerPrefix} Removed index ${indexName} from ${tableName}`)
+  } catch {
+    logger.info(`${loggerPrefix} Index ${indexName} does not exist or could not be removed`)
+  }
+}
+
+/**
+ * @param {MigrationOptions} options - an object containing the migration context.
+ * @returns {Promise<void>}
+ */
+async function up({ context: { queryInterface } }) {
+  const logger = require('../Logger')
+  const dialect = queryInterface.sequelize.getDialect()
+  logger.info(`${loggerPrefix} Running migration for dialect: ${dialect}`)
+
+  const booksExist = await tableExists(queryInterface, dialect, 'books')
+  if (!booksExist) {
+    logger.info(`${loggerPrefix} Core tables do not exist yet (fresh database), skipping migration`)
+    return
+  }
+
+  for (const index of indexesToCreate) {
+    await safeCreateIndex(queryInterface, logger, dialect, index.tableName, index.indexName, index.getFields(dialect))
+  }
+
+  logger.info(`${loggerPrefix} Migration completed successfully`)
+}
+
+/**
+ * @param {MigrationOptions} options - an object containing the migration context.
+ * @returns {Promise<void>}
+ */
+async function down({ context: { queryInterface } }) {
+  const logger = require('../Logger')
+  logger.info(`${loggerPrefix} Running downgrade migration...`)
+
+  for (const index of indexesToCreate) {
+    await safeRemoveIndex(queryInterface, logger, index.tableName, index.indexName)
+  }
+
+  logger.info(`${loggerPrefix} Downgrade migration completed`)
+}
+
+module.exports = {
+  up,
+  down,
+  name: migrationName
+}

test/server/migrations/v2.32.6-large-library-browse-indexes.postgres.test.js

@@ -0,0 +1,95 @@
+const chai = require('chai')
+const sinon = require('sinon')
+const { expect } = chai
+
+const { DataTypes, Sequelize } = require('sequelize')
+const Logger = require('../../../server/Logger')
+
+const { up } = require('../../../server/migrations/v2.32.6-large-library-browse-indexes')
+
+const TEST_POSTGRES_URL = process.env.TEST_POSTGRES_URL
+
+describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => {
+  if (!TEST_POSTGRES_URL) {
+    it('skips because TEST_POSTGRES_URL is not set', function() {
+      this.skip()
+    })
+    return
+  }
+
+  let sequelize
+  let queryInterface
+
+  beforeEach(async () => {
+    sequelize = new Sequelize(TEST_POSTGRES_URL, {
+      dialect: 'postgres',
+      logging: false
+    })
+    queryInterface = sequelize.getQueryInterface()
+    sinon.stub(Logger, 'info')
+
+    await queryInterface.dropTable('mediaProgresses').catch(() => {})
+    await queryInterface.dropTable('libraryItems').catch(() => {})
+    await queryInterface.dropTable('books').catch(() => {})
+
+    await queryInterface.createTable('books', {
+      id: { type: DataTypes.UUID, allowNull: false, primaryKey: true }
+    })
+
+    await queryInterface.createTable('libraryItems', {
+      id: { type: DataTypes.UUID, allowNull: false, primaryKey: true },
+      libraryId: { type: DataTypes.UUID, allowNull: false },
+      mediaType: { type: DataTypes.STRING, allowNull: false },
+      title: { type: DataTypes.TEXT, allowNull: true },
+      titleIgnorePrefix: { type: DataTypes.TEXT, allowNull: true },
+      createdAt: { type: DataTypes.DATE, allowNull: true }
+    })
+
+    await queryInterface.createTable('mediaProgresses', {
+      id: { type: DataTypes.UUID, allowNull: false, primaryKey: true },
+      userId: { type: DataTypes.UUID, allowNull: false },
+      updatedAt: { type: DataTypes.DATE, allowNull: true }
+    })
+  })
+
+  afterEach(async () => {
+    sinon.restore()
+    if (queryInterface) {
+      await queryInterface.dropTable('mediaProgresses').catch(() => {})
+      await queryInterface.dropTable('libraryItems').catch(() => {})
+      await queryInterface.dropTable('books').catch(() => {})
+    }
+    if (sequelize) {
+      await sequelize.close()
+    }
+  })
+
+  it('creates browse indexes with the expected names and column order', async () => {
+    await up({ context: { queryInterface, logger: Logger } })
+
+    const [rows] = await queryInterface.sequelize.query(`
+      SELECT indexname, indexdef
+      FROM pg_indexes
+      WHERE schemaname = 'public'
+        AND indexname IN (
+          'library_items_library_media_type_title_id',
+          'library_items_library_media_type_title_ignore_prefix_id',
+          'library_items_library_media_type_created_at_id',
+          'media_progress_user_updated_at_id'
+        )
+      ORDER BY indexname ASC
+    `)
+
+    expect(rows.map((row) => row.indexname)).to.deep.equal([
+      'library_items_library_media_type_created_at_id',
+      'library_items_library_media_type_title_id',
+      'library_items_library_media_type_title_ignore_prefix_id',
+      'media_progress_user_updated_at_id'
+    ])
+
+    expect(rows.find((row) => row.indexname === 'library_items_library_media_type_title_id').indexdef).to.include('("libraryId", "mediaType", title, id)')
+    expect(rows.find((row) => row.indexname === 'library_items_library_media_type_title_ignore_prefix_id').indexdef).to.include('("libraryId", "mediaType", "titleIgnorePrefix", id)')
+    expect(rows.find((row) => row.indexname === 'library_items_library_media_type_created_at_id').indexdef).to.include('("libraryId", "mediaType", "createdAt", id)')
+    expect(rows.find((row) => row.indexname === 'media_progress_user_updated_at_id').indexdef).to.include('("userId", "updatedAt", id)')
+  })
+})

test/server/migrations/v2.32.6-large-library-browse-indexes.test.js

@@ -0,0 +1,106 @@
+const chai = require('chai')
+const sinon = require('sinon')
+const { expect } = chai
+
+const { DataTypes, Sequelize } = require('sequelize')
+const Logger = require('../../../server/Logger')
+
+const { up } = require('../../../server/migrations/v2.32.6-large-library-browse-indexes')
+
+const normalizeWhitespace = (str) => str.replace(/\s+/g, ' ').trim().replace(/`/g, '')
+
+describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => {
+  let sequelize
+  let queryInterface
+
+  beforeEach(async () => {
+    sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
+    queryInterface = sequelize.getQueryInterface()
+    sinon.stub(Logger, 'info')
+
+    await queryInterface.createTable('books', {
+      id: { type: DataTypes.UUID, allowNull: false, primaryKey: true }
+    })
+
+    await queryInterface.createTable('libraryItems', {
+      id: { type: DataTypes.UUID, allowNull: false, primaryKey: true },
+      libraryId: { type: DataTypes.UUID, allowNull: false },
+      mediaType: { type: DataTypes.STRING, allowNull: false },
+      title: { type: DataTypes.TEXT, allowNull: true },
+      titleIgnorePrefix: { type: DataTypes.TEXT, allowNull: true },
+      createdAt: { type: DataTypes.DATE, allowNull: true }
+    })
+
+    await queryInterface.createTable('mediaProgresses', {
+      id: { type: DataTypes.UUID, allowNull: false, primaryKey: true },
+      userId: { type: DataTypes.UUID, allowNull: false },
+      updatedAt: { type: DataTypes.DATE, allowNull: true }
+    })
+  })
+
+  afterEach(async () => {
+    sinon.restore()
+    await sequelize.close()
+  })
+
+  it('creates browse indexes for title and progress keyset traversal', async () => {
+    await up({ context: { queryInterface, logger: Logger } })
+
+    const [[{ count: titleExactIndexCount }]] = await queryInterface.sequelize.query(
+      "SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_title_id'"
+    )
+    expect(titleExactIndexCount).to.equal(1)
+
+    const [[{ sql: titleExactIndexSql }]] = await queryInterface.sequelize.query(
+      "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_title_id'"
+    )
+    expect(normalizeWhitespace(titleExactIndexSql)).to.equal(
+      normalizeWhitespace('CREATE INDEX library_items_library_media_type_title_id ON libraryItems (libraryId, mediaType, title COLLATE NOCASE, id)')
+    )
+
+    const [[{ count: titleIndexCount }]] = await queryInterface.sequelize.query(
+      "SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_title_ignore_prefix_id'"
+    )
+    expect(titleIndexCount).to.equal(1)
+
+    const [[{ sql: titleIndexSql }]] = await queryInterface.sequelize.query(
+      "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_title_ignore_prefix_id'"
+    )
+    expect(normalizeWhitespace(titleIndexSql)).to.equal(
+      normalizeWhitespace('CREATE INDEX library_items_library_media_type_title_ignore_prefix_id ON libraryItems (libraryId, mediaType, titleIgnorePrefix COLLATE NOCASE, id)')
+    )
+
+    const [[{ sql: createdAtIndexSql }]] = await queryInterface.sequelize.query(
+      "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_created_at_id'"
+    )
+    expect(normalizeWhitespace(createdAtIndexSql)).to.equal(
+      normalizeWhitespace('CREATE INDEX library_items_library_media_type_created_at_id ON libraryItems (libraryId, mediaType, createdAt, id)')
+    )
+
+    const [[{ sql: progressIndexSql }]] = await queryInterface.sequelize.query(
+      "SELECT sql FROM sqlite_master WHERE type='index' AND name='media_progress_user_updated_at_id'"
+    )
+    expect(normalizeWhitespace(progressIndexSql)).to.equal(
+      normalizeWhitespace('CREATE INDEX media_progress_user_updated_at_id ON mediaProgresses (userId, updatedAt, id)')
+    )
+  })
+
+  it('is idempotent', async () => {
+    await up({ context: { queryInterface, logger: Logger } })
+    await up({ context: { queryInterface, logger: Logger } })
+
+    const indexes = [
+      'library_items_library_media_type_title_id',
+      'library_items_library_media_type_title_ignore_prefix_id',
+      'library_items_library_media_type_created_at_id',
+      'media_progress_user_updated_at_id'
+    ]
+
+    for (const indexName of indexes) {
+      const [[{ count }]] = await queryInterface.sequelize.query(
+        `SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='${indexName}'`
+      )
+      expect(count, indexName).to.equal(1)
+    }
+  })
+})