mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2026-07-07 09:51:37 +00:00
perf: fix large-library browse index coverage
This commit is contained in:
Jonathan Finley
parent
d39de74db0
commit
f1806e2a8c
3 changed files with 156 additions and 13 deletions
|
|
@ -15,15 +15,61 @@ const indexesToCreate = [
|
|||
{
|
||||
tableName: 'libraryItems',
|
||||
indexName: 'library_items_library_media_type_title_id',
|
||||
getFields(dialect) {
|
||||
return ['libraryId', 'mediaType', dialect === 'sqlite' ? { name: 'title', collate: 'NOCASE' } : 'title', 'id']
|
||||
getFields() {
|
||||
return ['libraryId', 'mediaType', { name: 'title', collate: 'NOCASE' }, 'id']
|
||||
},
|
||||
getPostgresSql() {
|
||||
return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_title_id ON "libraryItems" ("libraryId", "mediaType", LOWER("title"), "id")'
|
||||
}
|
||||
},
|
||||
{
|
||||
tableName: 'libraryItems',
|
||||
indexName: 'library_items_library_media_type_title_ignore_prefix_id',
|
||||
getFields(dialect) {
|
||||
return ['libraryId', 'mediaType', dialect === 'sqlite' ? { name: 'titleIgnorePrefix', collate: 'NOCASE' } : 'titleIgnorePrefix', 'id']
|
||||
getFields() {
|
||||
return ['libraryId', 'mediaType', { name: 'titleIgnorePrefix', collate: 'NOCASE' }, 'id']
|
||||
},
|
||||
getPostgresSql() {
|
||||
return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_title_ignore_prefix_id ON "libraryItems" ("libraryId", "mediaType", LOWER("titleIgnorePrefix"), "id")'
|
||||
}
|
||||
},
|
||||
{
|
||||
tableName: 'libraryItems',
|
||||
indexName: 'library_items_library_media_type_author_names_first_last_title_id',
|
||||
getFields() {
|
||||
return ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }, { name: 'title', collate: 'NOCASE' }, 'id']
|
||||
},
|
||||
getPostgresSql() {
|
||||
return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_first_last_title_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesFirstLast"), LOWER("title"), "id")'
|
||||
}
|
||||
},
|
||||
{
|
||||
tableName: 'libraryItems',
|
||||
indexName: 'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id',
|
||||
getFields() {
|
||||
return ['libraryId', 'mediaType', { name: 'authorNamesFirstLast', collate: 'NOCASE' }, { name: 'titleIgnorePrefix', collate: 'NOCASE' }, 'id']
|
||||
},
|
||||
getPostgresSql() {
|
||||
return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_first_last_title_ignore_prefix_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesFirstLast"), LOWER("titleIgnorePrefix"), "id")'
|
||||
}
|
||||
},
|
||||
{
|
||||
tableName: 'libraryItems',
|
||||
indexName: 'library_items_library_media_type_author_names_last_first_title_id',
|
||||
getFields() {
|
||||
return ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }, { name: 'title', collate: 'NOCASE' }, 'id']
|
||||
},
|
||||
getPostgresSql() {
|
||||
return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_last_first_title_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesLastFirst"), LOWER("title"), "id")'
|
||||
}
|
||||
},
|
||||
{
|
||||
tableName: 'libraryItems',
|
||||
indexName: 'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id',
|
||||
getFields() {
|
||||
return ['libraryId', 'mediaType', { name: 'authorNamesLastFirst', collate: 'NOCASE' }, { name: 'titleIgnorePrefix', collate: 'NOCASE' }, 'id']
|
||||
},
|
||||
getPostgresSql() {
|
||||
return 'CREATE INDEX IF NOT EXISTS library_items_library_media_type_author_names_last_first_title_ignore_prefix_id ON "libraryItems" ("libraryId", "mediaType", LOWER("authorNamesLastFirst"), LOWER("titleIgnorePrefix"), "id")'
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -33,6 +79,13 @@ const indexesToCreate = [
|
|||
return ['libraryId', 'mediaType', 'createdAt', 'id']
|
||||
}
|
||||
},
|
||||
{
|
||||
tableName: 'libraryItems',
|
||||
indexName: 'library_items_library_media_type_updated_at_id',
|
||||
getFields() {
|
||||
return ['libraryId', 'mediaType', 'updatedAt', 'id']
|
||||
}
|
||||
},
|
||||
{
|
||||
tableName: 'mediaProgresses',
|
||||
indexName: 'media_progress_user_updated_at_id',
|
||||
|
|
@ -79,6 +132,30 @@ async function safeCreateIndex(queryInterface, logger, dialect, tableName, index
|
|||
}
|
||||
}
|
||||
|
||||
async function safeCreatePostgresIndex(queryInterface, logger, dialect, tableName, indexName, sql) {
|
||||
try {
|
||||
if (!(await tableExists(queryInterface, dialect, tableName))) {
|
||||
logger.info(`${loggerPrefix} Table ${tableName} does not exist, skipping index ${indexName}`)
|
||||
return
|
||||
}
|
||||
|
||||
await queryInterface.sequelize.query(sql)
|
||||
logger.info(`${loggerPrefix} Created index ${indexName} on ${tableName}`)
|
||||
} catch (error) {
|
||||
if (error.message?.includes('already exists')) {
|
||||
logger.info(`${loggerPrefix} Index ${indexName} already exists, skipping`)
|
||||
return
|
||||
}
|
||||
|
||||
if (error.message?.includes('does not exist')) {
|
||||
logger.info(`${loggerPrefix} Table ${tableName} does not exist, skipping index ${indexName}`)
|
||||
return
|
||||
}
|
||||
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function safeRemoveIndex(queryInterface, logger, tableName, indexName) {
|
||||
try {
|
||||
await queryInterface.removeIndex(tableName, indexName)
|
||||
|
|
@ -104,8 +181,12 @@ async function up({ context: { queryInterface } }) {
|
|||
}
|
||||
|
||||
for (const index of indexesToCreate) {
|
||||
if (dialect === 'postgres' && index.getPostgresSql) {
|
||||
await safeCreatePostgresIndex(queryInterface, logger, dialect, index.tableName, index.indexName, index.getPostgresSql())
|
||||
} else {
|
||||
await safeCreateIndex(queryInterface, logger, dialect, index.tableName, index.indexName, index.getFields(dialect))
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`${loggerPrefix} Migration completed successfully`)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ const Logger = require('../../../server/Logger')
|
|||
const { up } = require('../../../server/migrations/v2.32.6-large-library-browse-indexes')
|
||||
|
||||
const TEST_POSTGRES_URL = process.env.TEST_POSTGRES_URL
|
||||
const normalizeWhitespace = (str) => str.replace(/\s+/g, ' ').trim().toLowerCase()
|
||||
|
||||
describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => {
|
||||
if (!TEST_POSTGRES_URL) {
|
||||
|
|
@ -42,7 +43,10 @@ describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => {
|
|||
mediaType: { type: DataTypes.STRING, allowNull: false },
|
||||
title: { type: DataTypes.TEXT, allowNull: true },
|
||||
titleIgnorePrefix: { type: DataTypes.TEXT, allowNull: true },
|
||||
createdAt: { type: DataTypes.DATE, allowNull: true }
|
||||
authorNamesFirstLast: { type: DataTypes.TEXT, allowNull: true },
|
||||
authorNamesLastFirst: { type: DataTypes.TEXT, allowNull: true },
|
||||
createdAt: { type: DataTypes.DATE, allowNull: true },
|
||||
updatedAt: { type: DataTypes.DATE, allowNull: true }
|
||||
})
|
||||
|
||||
await queryInterface.createTable('mediaProgresses', {
|
||||
|
|
@ -64,7 +68,7 @@ describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => {
|
|||
}
|
||||
})
|
||||
|
||||
it('creates browse indexes with the expected names and column order', async () => {
|
||||
it('creates browse indexes with the expected names and functional order', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [rows] = await queryInterface.sequelize.query(`
|
||||
|
|
@ -72,24 +76,39 @@ describe('Migration v2.32.6-large-library-browse-indexes (postgres)', () => {
|
|||
FROM pg_indexes
|
||||
WHERE schemaname = 'public'
|
||||
AND indexname IN (
|
||||
'library_items_library_media_type_author_names_first_last_title_id',
|
||||
'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id',
|
||||
'library_items_library_media_type_author_names_last_first_title_id',
|
||||
'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id',
|
||||
'library_items_library_media_type_title_id',
|
||||
'library_items_library_media_type_title_ignore_prefix_id',
|
||||
'library_items_library_media_type_created_at_id',
|
||||
'library_items_library_media_type_updated_at_id',
|
||||
'media_progress_user_updated_at_id'
|
||||
)
|
||||
ORDER BY indexname ASC
|
||||
`)
|
||||
|
||||
expect(rows.map((row) => row.indexname)).to.deep.equal([
|
||||
'library_items_library_media_type_author_names_first_last_title_id',
|
||||
'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id',
|
||||
'library_items_library_media_type_author_names_last_first_title_id',
|
||||
'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id',
|
||||
'library_items_library_media_type_created_at_id',
|
||||
'library_items_library_media_type_title_id',
|
||||
'library_items_library_media_type_title_ignore_prefix_id',
|
||||
'library_items_library_media_type_updated_at_id',
|
||||
'media_progress_user_updated_at_id'
|
||||
])
|
||||
|
||||
expect(rows.find((row) => row.indexname === 'library_items_library_media_type_title_id').indexdef).to.include('("libraryId", "mediaType", title, id)')
|
||||
expect(rows.find((row) => row.indexname === 'library_items_library_media_type_title_ignore_prefix_id').indexdef).to.include('("libraryId", "mediaType", "titleIgnorePrefix", id)')
|
||||
expect(rows.find((row) => row.indexname === 'library_items_library_media_type_created_at_id').indexdef).to.include('("libraryId", "mediaType", "createdAt", id)')
|
||||
expect(rows.find((row) => row.indexname === 'media_progress_user_updated_at_id').indexdef).to.include('("userId", "updatedAt", id)')
|
||||
expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_first_last_title_id').indexdef)).to.include('lower("authornamesfirstlast"), lower("title"), "id"')
|
||||
expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id').indexdef)).to.include('lower("authornamesfirstlast"), lower("titleignoreprefix"), "id"')
|
||||
expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_last_first_title_id').indexdef)).to.include('lower("authornameslastfirst"), lower("title"), "id"')
|
||||
expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id').indexdef)).to.include('lower("authornameslastfirst"), lower("titleignoreprefix"), "id"')
|
||||
expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_title_id').indexdef)).to.include('lower("title"), "id"')
|
||||
expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_title_ignore_prefix_id').indexdef)).to.include('lower("titleignoreprefix"), "id"')
|
||||
expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_created_at_id').indexdef)).to.include('("libraryid", "mediatype", "createdat", "id")')
|
||||
expect(normalizeWhitespace(rows.find((row) => row.indexname === 'library_items_library_media_type_updated_at_id').indexdef)).to.include('("libraryid", "mediatype", "updatedat", "id")')
|
||||
expect(normalizeWhitespace(rows.find((row) => row.indexname === 'media_progress_user_updated_at_id').indexdef)).to.include('("userid", "updatedat", "id")')
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => {
|
|||
mediaType: { type: DataTypes.STRING, allowNull: false },
|
||||
title: { type: DataTypes.TEXT, allowNull: true },
|
||||
titleIgnorePrefix: { type: DataTypes.TEXT, allowNull: true },
|
||||
createdAt: { type: DataTypes.DATE, allowNull: true }
|
||||
authorNamesFirstLast: { type: DataTypes.TEXT, allowNull: true },
|
||||
authorNamesLastFirst: { type: DataTypes.TEXT, allowNull: true },
|
||||
createdAt: { type: DataTypes.DATE, allowNull: true },
|
||||
updatedAt: { type: DataTypes.DATE, allowNull: true }
|
||||
})
|
||||
|
||||
await queryInterface.createTable('mediaProgresses', {
|
||||
|
|
@ -43,7 +46,7 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => {
|
|||
await sequelize.close()
|
||||
})
|
||||
|
||||
it('creates browse indexes for title and progress keyset traversal', async () => {
|
||||
it('creates browse indexes for title, author, timestamp, and progress traversal', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [[{ count: titleExactIndexCount }]] = await queryInterface.sequelize.query(
|
||||
|
|
@ -77,6 +80,41 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => {
|
|||
normalizeWhitespace('CREATE INDEX library_items_library_media_type_created_at_id ON libraryItems (libraryId, mediaType, createdAt, id)')
|
||||
)
|
||||
|
||||
const [[{ sql: updatedAtIndexSql }]] = await queryInterface.sequelize.query(
|
||||
"SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_updated_at_id'"
|
||||
)
|
||||
expect(normalizeWhitespace(updatedAtIndexSql)).to.equal(
|
||||
normalizeWhitespace('CREATE INDEX library_items_library_media_type_updated_at_id ON libraryItems (libraryId, mediaType, updatedAt, id)')
|
||||
)
|
||||
|
||||
const [[{ sql: authorIndexSql }]] = await queryInterface.sequelize.query(
|
||||
"SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_first_last_title_id'"
|
||||
)
|
||||
expect(normalizeWhitespace(authorIndexSql)).to.equal(
|
||||
normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_first_last_title_id ON libraryItems (libraryId, mediaType, authorNamesFirstLast COLLATE NOCASE, title COLLATE NOCASE, id)')
|
||||
)
|
||||
|
||||
const [[{ sql: authorIgnorePrefixIndexSql }]] = await queryInterface.sequelize.query(
|
||||
"SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_first_last_title_ignore_prefix_id'"
|
||||
)
|
||||
expect(normalizeWhitespace(authorIgnorePrefixIndexSql)).to.equal(
|
||||
normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_first_last_title_ignore_prefix_id ON libraryItems (libraryId, mediaType, authorNamesFirstLast COLLATE NOCASE, titleIgnorePrefix COLLATE NOCASE, id)')
|
||||
)
|
||||
|
||||
const [[{ sql: authorLfIndexSql }]] = await queryInterface.sequelize.query(
|
||||
"SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_last_first_title_id'"
|
||||
)
|
||||
expect(normalizeWhitespace(authorLfIndexSql)).to.equal(
|
||||
normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_last_first_title_id ON libraryItems (libraryId, mediaType, authorNamesLastFirst COLLATE NOCASE, title COLLATE NOCASE, id)')
|
||||
)
|
||||
|
||||
const [[{ sql: authorLfIgnorePrefixIndexSql }]] = await queryInterface.sequelize.query(
|
||||
"SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_library_media_type_author_names_last_first_title_ignore_prefix_id'"
|
||||
)
|
||||
expect(normalizeWhitespace(authorLfIgnorePrefixIndexSql)).to.equal(
|
||||
normalizeWhitespace('CREATE INDEX library_items_library_media_type_author_names_last_first_title_ignore_prefix_id ON libraryItems (libraryId, mediaType, authorNamesLastFirst COLLATE NOCASE, titleIgnorePrefix COLLATE NOCASE, id)')
|
||||
)
|
||||
|
||||
const [[{ sql: progressIndexSql }]] = await queryInterface.sequelize.query(
|
||||
"SELECT sql FROM sqlite_master WHERE type='index' AND name='media_progress_user_updated_at_id'"
|
||||
)
|
||||
|
|
@ -93,6 +131,11 @@ describe('Migration v2.32.6-large-library-browse-indexes (sqlite)', () => {
|
|||
'library_items_library_media_type_title_id',
|
||||
'library_items_library_media_type_title_ignore_prefix_id',
|
||||
'library_items_library_media_type_created_at_id',
|
||||
'library_items_library_media_type_updated_at_id',
|
||||
'library_items_library_media_type_author_names_first_last_title_id',
|
||||
'library_items_library_media_type_author_names_first_last_title_ignore_prefix_id',
|
||||
'library_items_library_media_type_author_names_last_first_title_id',
|
||||
'library_items_library_media_type_author_names_last_first_title_ignore_prefix_id',
|
||||
'media_progress_user_updated_at_id'
|
||||
]
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue