perf: fix large-library browse index coverage

server/migrations/v2.32.6-large-library-browse-indexes.js

@@ -15,15 +15,61 @@ const indexesToCreate = [
   {
     tableName: 'libraryItems',
     indexName: 'library_items_library_media_type_title_id',
-    getFields(dialect) {
+    getFields() {
-      return ['libraryId', 'mediaType', dialect === 'sqlite' ? { name: 'title', collate: 'NOCASE' } : 'title', 'id']
+      return ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }, 'id']
+    },
+    getPostgresSql() {
+      return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_title_id ON "libraryItems" ("libraryId", "mediaType", LOWER("title"), "id")'
     }
   },
   {
     tableName: 'libraryItems',
     indexName: 'library_items_library_media_type_title_ignore_prefix_id',
-    getFields(dialect) {
+    getFields() {
-      return ['libraryId', 'mediaType', dialect === 'sqlite' ? { name: 'titleIgnorePrefix', collate: 'NOCASE' } : 'titleIgnorePrefix', 'id']
+      return ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }, 'id']
+    },
+    getPostgresSql() {
+      return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_title_ignore_prefix_id ON "libraryItems" ("libraryId", "mediaType", LOWER("titleIgnorePrefix"), "id")'
+    }
+  },
+  {
+    tableName: 'libraryItems',
+    indexName: 'library_items_library_media_type_author_names_first_last_title_id',
+    getFields() {
+      return ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }, { name: 'title', collate: 'NOCASE' }, 'id']
+    },
+    getPostgresSql() {
+      return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_first_last_title_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesFirstLast"), LOWER("title"), "id")'
+    }
+  },
+  {
+    tableName: 'libraryItems',
+    indexName: 'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id',
+    getFields() {
+      return ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }, { name: 'titleIgnorePrefix', collate: 'NOCASE' }, 'id']
+    },
+    getPostgresSql() {
+      return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_first_last_title_ignore_prefix_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesFirstLast"), LOWER("titleIgnorePrefix"), "id")'
+    }
+  },
+  {
+    tableName: 'libraryItems',
+    indexName: 'library_items_library_media_type_author_names_last_first_title_id',
+    getFields() {
+      return ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }, { name: 'title', collate: 'NOCASE' }, 'id']
+    },
+    getPostgresSql() {
+      return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_last_first_title_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesLastFirst"), LOWER("title"), "id")'
+    }
+  },
+  {
+    tableName: 'libraryItems',
+    indexName: 'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id',
+    getFields() {
+      return ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }, { name: 'titleIgnorePrefix', collate: 'NOCASE' }, 'id']
+    },
+    getPostgresSql() {
+      return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_last_first_title_ignore_prefix_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesLastFirst"), LOWER("titleIgnorePrefix"), "id")'
     }
   },
   {
@@ -33,6 +79,13 @@ const indexesToCreate = [
       return ['libraryId', 'mediaType', 'createdAt', 'id']
     }
   },
+  {
+    tableName: 'libraryItems',
+    indexName: 'library_items_library_media_type_updated_at_id',
+    getFields() {
+      return ['libraryId', 'mediaType', 'updatedAt', 'id']
+    }
+  },
   {
     tableName: 'mediaProgresses',
     indexName: 'media_progress_user_updated_at_id',
@@ -79,6 +132,30 @@ async function safeCreateIndex(queryInterface, logger, dialect, tableName, index
   }
 }
 
+async function safeCreatePostgresIndex(queryInterface, logger, dialect, tableName, indexName, sql) {
+  try {
+    if (!(await tableExists(queryInterface, dialect, tableName))) {
+      logger.info(`${loggerPrefix} Table ${tableName} does not exist, skipping index ${indexName}`)
+      return
+    }
+
+    await queryInterface.sequelize.query(sql)
+    logger.info(`${loggerPrefix} Created index ${indexName} on ${tableName}`)
+  } catch (error) {
+    if (error.message?.includes('already exists')) {
+      logger.info(`${loggerPrefix} Index ${indexName} already exists, skipping`)
+      return
+    }
+
+    if (error.message?.includes('does not exist')) {
+      logger.info(`${loggerPrefix} Table ${tableName} does not exist, skipping index ${indexName}`)
+      return
+    }
+
+    throw error
+  }
+}
+
 async function safeRemoveIndex(queryInterface, logger, tableName, indexName) {
   try {
     await queryInterface.removeIndex(tableName, indexName)
@@ -104,8 +181,12 @@ async function up({ context: { queryInterface } }) {
   }
 
   for (const index of indexesToCreate) {
+    if (dialect === 'postgres' && index.getPostgresSql) {
+      await safeCreatePostgresIndex(queryInterface, logger, dialect, index.tableName, index.indexName, index.getPostgresSql())
+    } else {
       await safeCreateIndex(queryInterface, logger, dialect, index.tableName, index.indexName, index.getFields(dialect))
     }
+  }
 
   logger.info(`${loggerPrefix} Migration completed successfully`)
 }

test/server/migrations/v2.32.6-large-library-browse-indexes.postgres.test.js

@@ -8,6 +8,7 @@ const Logger = require('../../../server/Logger')
 const { up } = require('../../../server/migrations/v2.32.6-large-library-browse-indexes')
 
 const TEST_POSTGRES_URL = process.env.TEST_POSTGRES_URL
+const normalizeWhitespace = (str) => str.replace(/\s+/g, ' ').trim().toLowerCase()
 
 describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => {
   if (!TEST_POSTGRES_URL) {
@@ -42,7 +43,10 @@ describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => {
       mediaType: { type: DataTypes.STRING, allowNull: false },
       title: { type: DataTypes.TEXT, allowNull: true },
       titleIgnorePrefix: { type: DataTypes.TEXT, allowNull: true },
-      createdAt: { type: DataTypes.DATE, allowNull: true }
+      authorNamesFirstLast: { type: DataTypes.TEXT, allowNull: true },
+      authorNamesLastFirst: { type: DataTypes.TEXT, allowNull: true },
+      createdAt: { type: DataTypes.DATE, allowNull: true },
+      updatedAt: { type: DataTypes.DATE, allowNull: true }
     })
 
     await queryInterface.createTable('mediaProgresses', {
@@ -64,7 +68,7 @@ describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => {
     }
   })
 
-  it('creates browse indexes with the expected names and column order', async () => {
+  it('creates browse indexes with the expected names and functional order', async () => {
     await up({ context: { queryInterface, logger: Logger } })
 
     const [rows] = await queryInterface.sequelize.query(`
@@ -72,24 +76,39 @@ describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => {
       FROM pg_indexes
       WHERE schemaname = 'public'
         AND indexname IN (
+          'library_items_library_media_type_author_names_first_last_title_id',
+          'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id',
+          'library_items_library_media_type_author_names_last_first_title_id',
+          'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id',
           'library_items_library_media_type_title_id',
           'library_items_library_media_type_title_ignore_prefix_id',
           'library_items_library_media_type_created_at_id',
+          'library_items_library_media_type_updated_at_id',
           'media_progress_user_updated_at_id'
         )
       ORDER BY indexname ASC
     `)
 
     expect(rows.map((row) => row.indexname)).to.deep.equal([
+      'library_items_library_media_type_author_names_first_last_title_id',
+      'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id',
+      'library_items_library_media_type_author_names_last_first_title_id',
+      'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id',
       'library_items_library_media_type_created_at_id',
       'library_items_library_media_type_title_id',
       'library_items_library_media_type_title_ignore_prefix_id',
+      'library_items_library_media_type_updated_at_id',
       'media_progress_user_updated_at_id'
     ])
 
-    expect(rows.find((row) => row.indexname === 'library_items_library_media_type_title_id').indexdef).to.include('("libraryId", "mediaType", title, id)')
+    expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_first_last_title_id').indexdef)).to.include('lower("authornamesfirstlast"), lower("title"), "id"')
-    expect(rows.find((row) => row.indexname === 'library_items_library_media_type_title_ignore_prefix_id').indexdef).to.include('("libraryId", "mediaType", "titleIgnorePrefix", id)')
+    expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id').indexdef)).to.include('lower("authornamesfirstlast"), lower("titleignoreprefix"), "id"')
-    expect(rows.find((row) => row.indexname === 'library_items_library_media_type_created_at_id').indexdef).to.include('("libraryId", "mediaType", "createdAt", id)')
+    expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_last_first_title_id').indexdef)).to.include('lower("authornameslastfirst"), lower("title"), "id"')
-    expect(rows.find((row) => row.indexname === 'media_progress_user_updated_at_id').indexdef).to.include('("userId", "updatedAt", id)')
+    expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id').indexdef)).to.include('lower("authornameslastfirst"), lower("titleignoreprefix"), "id"')
+    expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_title_id').indexdef)).to.include('lower("title"), "id"')
+    expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_title_ignore_prefix_id').indexdef)).to.include('lower("titleignoreprefix"), "id"')
+    expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_created_at_id').indexdef)).to.include('("libraryid", "mediatype", "createdat", "id")')
+    expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_updated_at_id').indexdef)).to.include('("libraryid", "mediatype", "updatedat", "id")')
+    expect(normalizeWhitespace(rows.find((row) => row.indexname === 'media_progress_user_updated_at_id').indexdef)).to.include('("userid", "updatedat", "id")')
   })
 })

test/server/migrations/v2.32.6-large-library-browse-indexes.test.js

@@ -28,7 +28,10 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => {
       mediaType: { type: DataTypes.STRING, allowNull: false },
       title: { type: DataTypes.TEXT, allowNull: true },
       titleIgnorePrefix: { type: DataTypes.TEXT, allowNull: true },
-      createdAt: { type: DataTypes.DATE, allowNull: true }
+      authorNamesFirstLast: { type: DataTypes.TEXT, allowNull: true },
+      authorNamesLastFirst: { type: DataTypes.TEXT, allowNull: true },
+      createdAt: { type: DataTypes.DATE, allowNull: true },
+      updatedAt: { type: DataTypes.DATE, allowNull: true }
     })
 
     await queryInterface.createTable('mediaProgresses', {
@@ -43,7 +46,7 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => {
     await sequelize.close()
   })
 
-  it('creates browse indexes for title and progress keyset traversal', async () => {
+  it('creates browse indexes for title, author, timestamp, and progress traversal', async () => {
     await up({ context: { queryInterface, logger: Logger } })
 
     const [[{ count: titleExactIndexCount }]] = await queryInterface.sequelize.query(
@@ -77,6 +80,41 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => {
       normalizeWhitespace('CREATE INDEX library_items_library_media_type_created_at_id ON libraryItems (libraryId, mediaType, createdAt, id)')
     )
 
+    const [[{ sql: updatedAtIndexSql }]] = await queryInterface.sequelize.query(
+      "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_updated_at_id'"
+    )
+    expect(normalizeWhitespace(updatedAtIndexSql)).to.equal(
+      normalizeWhitespace('CREATE INDEX library_items_library_media_type_updated_at_id ON libraryItems (libraryId, mediaType, updatedAt, id)')
+    )
+
+    const [[{ sql: authorIndexSql }]] = await queryInterface.sequelize.query(
+      "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_first_last_title_id'"
+    )
+    expect(normalizeWhitespace(authorIndexSql)).to.equal(
+      normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_first_last_title_id ON libraryItems (libraryId, mediaType, authorNamesFirstLast COLLATE NOCASE, title COLLATE NOCASE, id)')
+    )
+
+    const [[{ sql: authorIgnorePrefixIndexSql }]] = await queryInterface.sequelize.query(
+      "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_first_last_title_ignore_prefix_id'"
+    )
+    expect(normalizeWhitespace(authorIgnorePrefixIndexSql)).to.equal(
+      normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_first_last_title_ignore_prefix_id ON libraryItems (libraryId, mediaType, authorNamesFirstLast COLLATE NOCASE, titleIgnorePrefix COLLATE NOCASE, id)')
+    )
+
+    const [[{ sql: authorLfIndexSql }]] = await queryInterface.sequelize.query(
+      "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_last_first_title_id'"
+    )
+    expect(normalizeWhitespace(authorLfIndexSql)).to.equal(
+      normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_last_first_title_id ON libraryItems (libraryId, mediaType, authorNamesLastFirst COLLATE NOCASE, title COLLATE NOCASE, id)')
+    )
+
+    const [[{ sql: authorLfIgnorePrefixIndexSql }]] = await queryInterface.sequelize.query(
+      "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_last_first_title_ignore_prefix_id'"
+    )
+    expect(normalizeWhitespace(authorLfIgnorePrefixIndexSql)).to.equal(
+      normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_last_first_title_ignore_prefix_id ON libraryItems (libraryId, mediaType, authorNamesLastFirst COLLATE NOCASE, titleIgnorePrefix COLLATE NOCASE, id)')
+    )
+
     const [[{ sql: progressIndexSql }]] = await queryInterface.sequelize.query(
       "SELECT sql FROM sqlite_master WHERE type='index' AND name='media_progress_user_updated_at_id'"
     )
@@ -93,6 +131,11 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => {
       'library_items_library_media_type_title_id',
       'library_items_library_media_type_title_ignore_prefix_id',
       'library_items_library_media_type_created_at_id',
+      'library_items_library_media_type_updated_at_id',
+      'library_items_library_media_type_author_names_first_last_title_id',
+      'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id',
+      'library_items_library_media_type_author_names_last_first_title_id',
+      'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id',
       'media_progress_user_updated_at_id'
     ]