diff --git a/server/migrations/v2.32.6-large-library-browse-indexes.js b/server/migrations/v2.32.6-large-library-browse-indexes.js index 01b58be11..0044cac19 100644 --- a/server/migrations/v2.32.6-large-library-browse-indexes.js +++ b/server/migrations/v2.32.6-large-library-browse-indexes.js @@ -15,15 +15,61 @@ const indexesToCreate = [ { tableName: 'libraryItems', indexName: 'library_items_library_media_type_title_id', - getFields(dialect) { - return ['libraryId', 'mediaType', dialect === 'sqlite' ? { name: 'title', collate: 'NOCASE' } : 'title', 'id'] + getFields() { + return ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }, 'id'] + }, + getPostgresSql() { + return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_title_id ON "libraryItems" ("libraryId", "mediaType", LOWER("title"), "id")' } }, { tableName: 'libraryItems', indexName: 'library_items_library_media_type_title_ignore_prefix_id', - getFields(dialect) { - return ['libraryId', 'mediaType', dialect === 'sqlite' ? { name: 'titleIgnorePrefix', collate: 'NOCASE' } : 'titleIgnorePrefix', 'id'] + getFields() { + return ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }, 'id'] + }, + getPostgresSql() { + return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_title_ignore_prefix_id ON "libraryItems" ("libraryId", "mediaType", LOWER("titleIgnorePrefix"), "id")' + } + }, + { + tableName: 'libraryItems', + indexName: 'library_items_library_media_type_author_names_first_last_title_id', + getFields() { + return ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }, { name: 'title', collate: 'NOCASE' }, 'id'] + }, + getPostgresSql() { + return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_first_last_title_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesFirstLast"), LOWER("title"), "id")' + } + }, + { + tableName: 'libraryItems', + indexName: 'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id', + getFields() { + return ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }, { name: 'titleIgnorePrefix', collate: 'NOCASE' }, 'id'] + }, + getPostgresSql() { + return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_first_last_title_ignore_prefix_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesFirstLast"), LOWER("titleIgnorePrefix"), "id")' + } + }, + { + tableName: 'libraryItems', + indexName: 'library_items_library_media_type_author_names_last_first_title_id', + getFields() { + return ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }, { name: 'title', collate: 'NOCASE' }, 'id'] + }, + getPostgresSql() { + return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_last_first_title_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesLastFirst"), LOWER("title"), "id")' + } + }, + { + tableName: 'libraryItems', + indexName: 'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id', + getFields() { + return ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }, { name: 'titleIgnorePrefix', collate: 'NOCASE' }, 'id'] + }, + getPostgresSql() { + return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_last_first_title_ignore_prefix_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesLastFirst"), LOWER("titleIgnorePrefix"), "id")' } }, { @@ -33,6 +79,13 @@ const indexesToCreate = [ return ['libraryId', 'mediaType', 'createdAt', 'id'] } }, + { + tableName: 'libraryItems', + indexName: 'library_items_library_media_type_updated_at_id', + getFields() { + return ['libraryId', 'mediaType', 'updatedAt', 'id'] + } + }, { tableName: 'mediaProgresses', indexName: 'media_progress_user_updated_at_id', @@ -79,6 +132,30 @@ async function safeCreateIndex(queryInterface, logger, dialect, tableName, index } } +async function safeCreatePostgresIndex(queryInterface, logger, dialect, tableName, indexName, sql) { + try { + if (!(await tableExists(queryInterface, dialect, tableName))) { + logger.info(`${loggerPrefix} Table ${tableName} does not exist, skipping index ${indexName}`) + return + } + + await queryInterface.sequelize.query(sql) + logger.info(`${loggerPrefix} Created index ${indexName} on ${tableName}`) + } catch (error) { + if (error.message?.includes('already exists')) { + logger.info(`${loggerPrefix} Index ${indexName} already exists, skipping`) + return + } + + if (error.message?.includes('does not exist')) { + logger.info(`${loggerPrefix} Table ${tableName} does not exist, skipping index ${indexName}`) + return + } + + throw error + } +} + async function safeRemoveIndex(queryInterface, logger, tableName, indexName) { try { await queryInterface.removeIndex(tableName, indexName) @@ -104,7 +181,11 @@ async function up({ context: { queryInterface } }) { } for (const index of indexesToCreate) { - await safeCreateIndex(queryInterface, logger, dialect, index.tableName, index.indexName, index.getFields(dialect)) + if (dialect === 'postgres' && index.getPostgresSql) { + await safeCreatePostgresIndex(queryInterface, logger, dialect, index.tableName, index.indexName, index.getPostgresSql()) + } else { + await safeCreateIndex(queryInterface, logger, dialect, index.tableName, index.indexName, index.getFields(dialect)) + } } logger.info(`${loggerPrefix} Migration completed successfully`) diff --git a/test/server/migrations/v2.32.6-large-library-browse-indexes.postgres.test.js b/test/server/migrations/v2.32.6-large-library-browse-indexes.postgres.test.js index 37f7234d2..af4c09e41 100644 --- a/test/server/migrations/v2.32.6-large-library-browse-indexes.postgres.test.js +++ b/test/server/migrations/v2.32.6-large-library-browse-indexes.postgres.test.js @@ -8,6 +8,7 @@ const Logger = require('../../../server/Logger') const { up } = require('../../../server/migrations/v2.32.6-large-library-browse-indexes') const TEST_POSTGRES_URL = process.env.TEST_POSTGRES_URL +const normalizeWhitespace = (str) => str.replace(/\s+/g, ' ').trim().toLowerCase() describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => { if (!TEST_POSTGRES_URL) { @@ -42,7 +43,10 @@ describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => { mediaType: { type: DataTypes.STRING, allowNull: false }, title: { type: DataTypes.TEXT, allowNull: true }, titleIgnorePrefix: { type: DataTypes.TEXT, allowNull: true }, - createdAt: { type: DataTypes.DATE, allowNull: true } + authorNamesFirstLast: { type: DataTypes.TEXT, allowNull: true }, + authorNamesLastFirst: { type: DataTypes.TEXT, allowNull: true }, + createdAt: { type: DataTypes.DATE, allowNull: true }, + updatedAt: { type: DataTypes.DATE, allowNull: true } }) await queryInterface.createTable('mediaProgresses', { @@ -64,7 +68,7 @@ describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => { } }) - it('creates browse indexes with the expected names and column order', async () => { + it('creates browse indexes with the expected names and functional order', async () => { await up({ context: { queryInterface, logger: Logger } }) const [rows] = await queryInterface.sequelize.query(` @@ -72,24 +76,39 @@ describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => { FROM pg_indexes WHERE schemaname = 'public' AND indexname IN ( + 'library_items_library_media_type_author_names_first_last_title_id', + 'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id', + 'library_items_library_media_type_author_names_last_first_title_id', + 'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id', 'library_items_library_media_type_title_id', 'library_items_library_media_type_title_ignore_prefix_id', 'library_items_library_media_type_created_at_id', + 'library_items_library_media_type_updated_at_id', 'media_progress_user_updated_at_id' ) ORDER BY indexname ASC `) expect(rows.map((row) => row.indexname)).to.deep.equal([ + 'library_items_library_media_type_author_names_first_last_title_id', + 'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id', + 'library_items_library_media_type_author_names_last_first_title_id', + 'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id', 'library_items_library_media_type_created_at_id', 'library_items_library_media_type_title_id', 'library_items_library_media_type_title_ignore_prefix_id', + 'library_items_library_media_type_updated_at_id', 'media_progress_user_updated_at_id' ]) - expect(rows.find((row) => row.indexname === 'library_items_library_media_type_title_id').indexdef).to.include('("libraryId", "mediaType", title, id)') - expect(rows.find((row) => row.indexname === 'library_items_library_media_type_title_ignore_prefix_id').indexdef).to.include('("libraryId", "mediaType", "titleIgnorePrefix", id)') - expect(rows.find((row) => row.indexname === 'library_items_library_media_type_created_at_id').indexdef).to.include('("libraryId", "mediaType", "createdAt", id)') - expect(rows.find((row) => row.indexname === 'media_progress_user_updated_at_id').indexdef).to.include('("userId", "updatedAt", id)') + expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_first_last_title_id').indexdef)).to.include('lower("authornamesfirstlast"), lower("title"), "id"') + expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id').indexdef)).to.include('lower("authornamesfirstlast"), lower("titleignoreprefix"), "id"') + expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_last_first_title_id').indexdef)).to.include('lower("authornameslastfirst"), lower("title"), "id"') + expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id').indexdef)).to.include('lower("authornameslastfirst"), lower("titleignoreprefix"), "id"') + expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_title_id').indexdef)).to.include('lower("title"), "id"') + expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_title_ignore_prefix_id').indexdef)).to.include('lower("titleignoreprefix"), "id"') + expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_created_at_id').indexdef)).to.include('("libraryid", "mediatype", "createdat", "id")') + expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_updated_at_id').indexdef)).to.include('("libraryid", "mediatype", "updatedat", "id")') + expect(normalizeWhitespace(rows.find((row) => row.indexname === 'media_progress_user_updated_at_id').indexdef)).to.include('("userid", "updatedat", "id")') }) }) diff --git a/test/server/migrations/v2.32.6-large-library-browse-indexes.test.js b/test/server/migrations/v2.32.6-large-library-browse-indexes.test.js index 64b54a41e..b6d34a174 100644 --- a/test/server/migrations/v2.32.6-large-library-browse-indexes.test.js +++ b/test/server/migrations/v2.32.6-large-library-browse-indexes.test.js @@ -28,7 +28,10 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => { mediaType: { type: DataTypes.STRING, allowNull: false }, title: { type: DataTypes.TEXT, allowNull: true }, titleIgnorePrefix: { type: DataTypes.TEXT, allowNull: true }, - createdAt: { type: DataTypes.DATE, allowNull: true } + authorNamesFirstLast: { type: DataTypes.TEXT, allowNull: true }, + authorNamesLastFirst: { type: DataTypes.TEXT, allowNull: true }, + createdAt: { type: DataTypes.DATE, allowNull: true }, + updatedAt: { type: DataTypes.DATE, allowNull: true } }) await queryInterface.createTable('mediaProgresses', { @@ -43,7 +46,7 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => { await sequelize.close() }) - it('creates browse indexes for title and progress keyset traversal', async () => { + it('creates browse indexes for title, author, timestamp, and progress traversal', async () => { await up({ context: { queryInterface, logger: Logger } }) const [[{ count: titleExactIndexCount }]] = await queryInterface.sequelize.query( @@ -77,6 +80,41 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => { normalizeWhitespace('CREATE INDEX library_items_library_media_type_created_at_id ON libraryItems (libraryId, mediaType, createdAt, id)') ) + const [[{ sql: updatedAtIndexSql }]] = await queryInterface.sequelize.query( + "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_updated_at_id'" + ) + expect(normalizeWhitespace(updatedAtIndexSql)).to.equal( + normalizeWhitespace('CREATE INDEX library_items_library_media_type_updated_at_id ON libraryItems (libraryId, mediaType, updatedAt, id)') + ) + + const [[{ sql: authorIndexSql }]] = await queryInterface.sequelize.query( + "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_first_last_title_id'" + ) + expect(normalizeWhitespace(authorIndexSql)).to.equal( + normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_first_last_title_id ON libraryItems (libraryId, mediaType, authorNamesFirstLast COLLATE NOCASE, title COLLATE NOCASE, id)') + ) + + const [[{ sql: authorIgnorePrefixIndexSql }]] = await queryInterface.sequelize.query( + "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_first_last_title_ignore_prefix_id'" + ) + expect(normalizeWhitespace(authorIgnorePrefixIndexSql)).to.equal( + normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_first_last_title_ignore_prefix_id ON libraryItems (libraryId, mediaType, authorNamesFirstLast COLLATE NOCASE, titleIgnorePrefix COLLATE NOCASE, id)') + ) + + const [[{ sql: authorLfIndexSql }]] = await queryInterface.sequelize.query( + "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_last_first_title_id'" + ) + expect(normalizeWhitespace(authorLfIndexSql)).to.equal( + normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_last_first_title_id ON libraryItems (libraryId, mediaType, authorNamesLastFirst COLLATE NOCASE, title COLLATE NOCASE, id)') + ) + + const [[{ sql: authorLfIgnorePrefixIndexSql }]] = await queryInterface.sequelize.query( + "SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_last_first_title_ignore_prefix_id'" + ) + expect(normalizeWhitespace(authorLfIgnorePrefixIndexSql)).to.equal( + normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_last_first_title_ignore_prefix_id ON libraryItems (libraryId, mediaType, authorNamesLastFirst COLLATE NOCASE, titleIgnorePrefix COLLATE NOCASE, id)') + ) + const [[{ sql: progressIndexSql }]] = await queryInterface.sequelize.query( "SELECT sql FROM sqlite_master WHERE type='index' AND name='media_progress_user_updated_at_id'" ) @@ -93,6 +131,11 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => { 'library_items_library_media_type_title_id', 'library_items_library_media_type_title_ignore_prefix_id', 'library_items_library_media_type_created_at_id', + 'library_items_library_media_type_updated_at_id', + 'library_items_library_media_type_author_names_first_last_title_id', + 'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id', + 'library_items_library_media_type_author_names_last_first_title_id', + 'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id', 'media_progress_user_updated_at_id' ]