From 59b81a76bb6cfb92e050df1faed19b720ee4e002 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Mon, 11 May 2026 19:40:46 -0700 Subject: [PATCH 01/12] Add initial Author name normalization and calculated column updates --- server/models/Author.js | 86 ++++++++++++++++++++++++++++++++------- server/scanner/Scanner.js | 14 +++---- 2 files changed, 77 insertions(+), 23 deletions(-) diff --git a/server/models/Author.js b/server/models/Author.js index 65561e211..aedeb8777 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -1,4 +1,4 @@ -const { DataTypes, Model, where, fn, col } = require('sequelize') +const { DataTypes, Model } = require('sequelize') const parseNameString = require('../utils/parsers/parseNameString') class Author extends Model { @@ -12,6 +12,8 @@ class Author extends Model { /** @type {string} */ this.lastFirst /** @type {string} */ + this.searchName + /** @type {string} */ this.asin /** @type {string} */ this.description @@ -35,6 +37,35 @@ class Author extends Model { return parseNameString.nameToLastFirst(name) } + static normalizeSearchName(name) { + if (!name?.trim()) return null + return name + .normalize('NFKC') // Standardize compatibility characters + .normalize('NFD') // Split accents into combining marks + .toLocaleLowerCase('und') + .replace(/[\p{P}\p{Z}\p{M}\s]+/gu, '') // Remove punctuation, whitespace, and diacritics + .trim() + } + + static buildAuthorDerivedFields(name) { + const searchName = this.normalizeSearchName(name) + if (!searchName) { + return { + lastFirst: null, + searchName: null + } + } + + return { + lastFirst: parseNameString.nameToLastFirst(name), + searchName + } + } + + static isAuthorNameMatch(leftName, rightName) { + return this.normalizeSearchName(leftName) === this.normalizeSearchName(rightName) + } + /** * Check if author exists * @param {string} authorId @@ -53,13 +84,13 @@ class Author extends Model { * @returns {Promise} */ static async getByNameAndLibrary(authorName, libraryId) { + const searchName = this.normalizeSearchName(authorName) + if (!searchName) return null return this.findOne({ - where: [ - where(fn('lower', col('name')), authorName.toLowerCase()), - { - libraryId - } - ] + where: { + searchName, + libraryId + } }) } @@ -114,14 +145,23 @@ class Author extends Model { * @returns {Promise<{ author: Author, created: boolean }>} */ static async findOrCreateByNameAndLibrary(name, libraryId) { - const author = await this.getByNameAndLibrary(name, libraryId) - if (author) return { author, created: false } - const newAuthor = await this.create({ - name, - lastFirst: this.getLastFirst(name), - libraryId + const searchName = this.normalizeSearchName(name) + if (!searchName) { + return { author: null, created: false } + } + + const [author, created] = await this.findOrCreate({ + where: { + searchName, + libraryId + }, + defaults: { + name, + libraryId, + ...this.buildAuthorDerivedFields(name) + } }) - return { author: newAuthor, created: true } + return { author, created } } /** @@ -138,6 +178,7 @@ class Author extends Model { }, name: DataTypes.STRING, lastFirst: DataTypes.STRING, + searchName: DataTypes.STRING, asin: DataTypes.STRING, description: DataTypes.TEXT, imagePath: DataTypes.STRING @@ -160,6 +201,19 @@ class Author extends Model { // collate: 'NOCASE' // }] // }, + { + fields: [ + { + name: 'searchName', + collate: 'NOCASE' + } + ] + }, + { + fields: ['searchName', 'libraryId'], + unique: true, + name: 'unique_author_search_name_per_library' + }, { fields: ['libraryId'] } @@ -167,6 +221,10 @@ class Author extends Model { } ) + Author.beforeSave((author) => { + Object.assign(author, Author.buildAuthorDerivedFields(author.name)) + }) + const { library } = sequelize.models library.hasMany(Author, { onDelete: 'CASCADE' diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js index af4405987..ec466300d 100644 --- a/server/scanner/Scanner.js +++ b/server/scanner/Scanner.js @@ -247,15 +247,11 @@ class Scanner { } const authorIdsRemoved = [] for (const authorName of matchData.author) { - const existingAuthor = libraryItem.media.authors.find((a) => a.name.toLowerCase() === authorName.toLowerCase()) + const existingAuthor = libraryItem.media.authors.find((a) => Database.authorModel.isAuthorNameMatch(a.name, authorName)) if (!existingAuthor) { - let author = await Database.authorModel.getByNameAndLibrary(authorName, libraryItem.libraryId) - if (!author) { - author = await Database.authorModel.create({ - name: authorName, - lastFirst: Database.authorModel.getLastFirst(authorName), - libraryId: libraryItem.libraryId - }) + const { author, created: isCreated } = await Database.authorModel.findOrCreateByNameAndLibrary(authorName, libraryItem.libraryId) + if (!author) continue + if (isCreated) { SocketAuthority.emitter('author_added', author.toOldJSON()) // Update filter data Database.addAuthorToFilterData(libraryItem.libraryId, author.name, author.id) @@ -271,7 +267,7 @@ class Scanner { hasAuthorUpdates = true }) } - const authorsRemoved = libraryItem.media.authors.filter((a) => !matchData.author.find((ma) => ma.toLowerCase() === a.name.toLowerCase())) + const authorsRemoved = libraryItem.media.authors.filter((a) => !matchData.author.find((ma) => Database.authorModel.isAuthorNameMatch(ma, a.name))) if (authorsRemoved.length) { for (const author of authorsRemoved) { await Database.bookAuthorModel.destroy({ where: { authorId: author.id, bookId: libraryItem.media.id } }) From 9b719b213a2224163e35d0d0605e447af7fed877 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Mon, 11 May 2026 19:44:25 -0700 Subject: [PATCH 02/12] Update author finding and creation for books --- server/models/Book.js | 6 +++--- server/scanner/BookScanner.js | 19 +++++++------------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/server/models/Book.js b/server/models/Book.js index d9f2ff132..d61ec3a46 100644 --- a/server/models/Book.js +++ b/server/models/Book.js @@ -465,9 +465,8 @@ class Book extends Model { /** @type {typeof import('./BookAuthor')} */ const bookAuthorModel = this.sequelize.models.bookAuthor - const authorsCleaned = authors.map((a) => a.toLowerCase()).filter((a) => a) - const authorsRemoved = this.authors.filter((au) => !authorsCleaned.includes(au.name.toLowerCase())) - const newAuthorNames = authors.filter((a) => !this.authors.some((au) => au.name.toLowerCase() === a.toLowerCase())) + const authorsRemoved = this.authors.filter((au) => !authors.some((authorName) => authorModel.isAuthorNameMatch(authorName, au.name))) + const newAuthorNames = authors.filter((a) => !this.authors.some((au) => authorModel.isAuthorNameMatch(au.name, a))) for (const author of authorsRemoved) { await bookAuthorModel.removeByIds(author.id, this.id) @@ -481,6 +480,7 @@ class Book extends Model { const authorsAdded = [] for (const authorName of newAuthorNames) { const { author, created } = await authorModel.findOrCreateByNameAndLibrary(authorName, libraryId) + if (!author) continue await bookAuthorModel.create({ bookId: this.id, authorId: author.id }) if (created) { SocketAuthority.emitter('author_added', author.toOldJSON()) diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js index ac93c6379..764856419 100644 --- a/server/scanner/BookScanner.js +++ b/server/scanner/BookScanner.js @@ -221,23 +221,18 @@ class BookScanner { if (key === 'authors') { // Check for authors added for (const authorName of bookMetadata.authors) { - if (!media.authors.some((au) => au.name === authorName)) { - const existingAuthorId = await Database.getAuthorIdByName(libraryItemData.libraryId, authorName) - if (existingAuthorId) { + if (!media.authors.some((au) => Database.authorModel.isAuthorNameMatch(au.name, authorName))) { + const { author, created } = await Database.authorModel.findOrCreateByNameAndLibrary(authorName, libraryItemData.libraryId) + if (!created) { await Database.bookAuthorModel.create({ bookId: media.id, - authorId: existingAuthorId + authorId: author.id }) libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added author "${authorName}"`) authorsUpdated = true } else { - const newAuthor = await Database.authorModel.create({ - name: authorName, - lastFirst: Database.authorModel.getLastFirst(authorName), - libraryId: libraryItemData.libraryId - }) - await media.addAuthor(newAuthor) - Database.addAuthorToFilterData(libraryItemData.libraryId, newAuthor.name, newAuthor.id) + await media.addAuthor(author) + Database.addAuthorToFilterData(libraryItemData.libraryId, author.name, author.id) libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new author "${authorName}"`) authorsUpdated = true } @@ -245,7 +240,7 @@ class BookScanner { } // Check for authors removed for (const author of media.authors) { - if (!bookMetadata.authors.includes(author.name)) { + if (!bookMetadata.authors.some((authorName) => Database.authorModel.isAuthorNameMatch(authorName, author.name))) { await author.bookAuthor.destroy() libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" removed author "${author.name}"`) authorsUpdated = true From eb8b575282adc87aca33493afc2730c526c92af8 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Mon, 11 May 2026 20:07:05 -0700 Subject: [PATCH 03/12] Database clean author computed columns at startup --- server/Database.js | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/server/Database.js b/server/Database.js index 213c2c61b..316880191 100644 --- a/server/Database.js +++ b/server/Database.js @@ -203,12 +203,44 @@ class Database { await this.addTriggers() await this.loadData() + await this.rebuildAuthorRows() Logger.info(`[Database] running ANALYZE`) await this.sequelize.query('ANALYZE') Logger.info(`[Database] ANALYZE completed`) } + /** + * Rebuild all author rows so derived fields stay in sync with the model logic. + */ + async rebuildAuthorRows() { + const pageSize = 500 + let offset = 0 + + while (true) { + const authors = await this.authorModel.findAll({ + attributes: ['id', 'name', 'libraryId'], + order: [['id', 'ASC']], + limit: pageSize, + offset + }) + + if (!authors.length) break + + for (const authorRef of authors) { + const author = await this.authorModel.findByPk(authorRef.id) + if (!author) continue + author.changed('name', true) + await author.save({ silent: true }) + } + + offset += authors.length + if (authors.length < pageSize) break + } + + Logger.debug(`Sanitized ${offset} author rows`) + } + /** * Connect to db * @returns {boolean} @@ -624,7 +656,9 @@ class Database { if (!this.libraryFilterData[libraryId]) { return (await this.authorModel.getByNameAndLibrary(authorName, libraryId))?.id || null } - return this.libraryFilterData[libraryId].authors.find((au) => au.name === authorName)?.id || null + const searchName = this.authorModel.normalizeSearchName(authorName) + if (!searchName) return null + return this.libraryFilterData[libraryId].authors.find((au) => this.authorModel.normalizeSearchName(au.name) === searchName)?.id || null } /** From e530cd4c6fabb53a6fba6f2616ea83268bcadc4a Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Wed, 13 May 2026 14:16:30 -0700 Subject: [PATCH 04/12] Add logging when author normalization is empty --- server/models/Book.js | 5 ++++- server/scanner/BookScanner.js | 4 ++++ server/scanner/Scanner.js | 5 ++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/server/models/Book.js b/server/models/Book.js index d61ec3a46..5c0fc10d4 100644 --- a/server/models/Book.js +++ b/server/models/Book.js @@ -480,7 +480,10 @@ class Book extends Model { const authorsAdded = [] for (const authorName of newAuthorNames) { const { author, created } = await authorModel.findOrCreateByNameAndLibrary(authorName, libraryId) - if (!author) continue + if (!author) { + Logger.warn(`[Book] "${this.title}" skipped author "${authorName}" because normalized name was empty`) + continue + } await bookAuthorModel.create({ bookId: this.id, authorId: author.id }) if (created) { SocketAuthority.emitter('author_added', author.toOldJSON()) diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js index 764856419..f859fd98b 100644 --- a/server/scanner/BookScanner.js +++ b/server/scanner/BookScanner.js @@ -223,6 +223,10 @@ class BookScanner { for (const authorName of bookMetadata.authors) { if (!media.authors.some((au) => Database.authorModel.isAuthorNameMatch(au.name, authorName))) { const { author, created } = await Database.authorModel.findOrCreateByNameAndLibrary(authorName, libraryItemData.libraryId) + if (!author) { + libraryScan.addLog(LogLevel.WARN, `Skipping author "${authorName}" because normalized name was empty`) + continue + } if (!created) { await Database.bookAuthorModel.create({ bookId: media.id, diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js index ec466300d..a4125e9ec 100644 --- a/server/scanner/Scanner.js +++ b/server/scanner/Scanner.js @@ -250,7 +250,10 @@ class Scanner { const existingAuthor = libraryItem.media.authors.find((a) => Database.authorModel.isAuthorNameMatch(a.name, authorName)) if (!existingAuthor) { const { author, created: isCreated } = await Database.authorModel.findOrCreateByNameAndLibrary(authorName, libraryItem.libraryId) - if (!author) continue + if (!author) { + libraryScan.addLog(LogLevel.WARN, `Skipping author "${authorName}" because normalized name was empty`) + continue + } if (isCreated) { SocketAuthority.emitter('author_added', author.toOldJSON()) // Update filter data From e61f2b98237e9dd8081c2d2a5c054906ff7eae16 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Wed, 13 May 2026 14:42:39 -0700 Subject: [PATCH 05/12] Initial migration script and associated test --- .../v2.34.0-add-author-search-name.js | 321 ++++++++++++++++++ .../v2.34.0-add-author-search-name.test.js | 255 ++++++++++++++ 2 files changed, 576 insertions(+) create mode 100644 server/migrations/v2.34.0-add-author-search-name.js create mode 100644 test/server/migrations/v2.34.0-add-author-search-name.test.js diff --git a/server/migrations/v2.34.0-add-author-search-name.js b/server/migrations/v2.34.0-add-author-search-name.js new file mode 100644 index 000000000..ddac8d645 --- /dev/null +++ b/server/migrations/v2.34.0-add-author-search-name.js @@ -0,0 +1,321 @@ +const { Sequelize } = require('sequelize') + +/** + * @typedef MigrationContext + * @property {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object. + * @property {import('../Logger')} logger - a Logger object. + * + * @typedef MigrationOptions + * @property {MigrationContext} context - an object containing the migration context. + */ + +const Author = require('../models/Author') + +const migrationVersion = '2.34.0' +const migrationName = `${migrationVersion}-add-author-search-name` +const loggerPrefix = `[${migrationVersion} migration]` +const AUTHORS_TABLE = 'authors' +const BOOK_AUTHORS_TABLE = 'bookAuthors' +const LIBRARY_ITEMS_TABLE = 'libraryItems' +const AUTHOR_SEARCH_INDEX = 'author_search_name' +const AUTHOR_LAST_FIRST_INDEX = 'author_last_first' +const UNIQUE_SEARCH_INDEX = 'unique_author_search_name_per_library' + +async function indexExists(queryInterface, tableName, indexName) { + const indexes = await queryInterface.showIndex(tableName) + return indexes.some((index) => index.name === indexName) +} + +async function addIndexIfMissing(queryInterface, logger, tableName, indexName, options, transaction = null) { + if (await indexExists(queryInterface, tableName, indexName)) { + logger.info(`${loggerPrefix} index "${indexName}" already exists on "${tableName}"`) + return + } + + logger.info(`${loggerPrefix} adding index "${indexName}" on "${tableName}"`) + await queryInterface.addIndex(tableName, options.fields, { + ...options, + name: indexName, + transaction + }) +} + +async function removeIndexIfPresent(queryInterface, logger, tableName, indexName) { + if (!(await indexExists(queryInterface, tableName, indexName))) { + logger.info(`${loggerPrefix} index "${indexName}" does not exist on "${tableName}"`) + return + } + + logger.info(`${loggerPrefix} removing index "${indexName}" from "${tableName}"`) + await queryInterface.removeIndex(tableName, indexName) +} + +async function backfillAuthorSearchName(queryInterface, logger, transaction, offset = 0) { + while (true) { + const authors = await queryInterface.sequelize.query(`SELECT id, name FROM ${AUTHORS_TABLE} ORDER BY id ASC LIMIT :limit OFFSET :offset`, { + replacements: { limit: 500, offset }, + type: Sequelize.QueryTypes.SELECT, + transaction + }) + + if (!authors.length) return + + logger.info(`${loggerPrefix} backfilling derived author fields for ${authors.length} authors`) + for (const author of authors) { + const derivedFields = Author.buildAuthorDerivedFields(author.name) + await queryInterface.sequelize.query( + `UPDATE ${AUTHORS_TABLE} + SET lastFirst = :lastFirst, + searchName = :searchName + WHERE id = :id`, + { + replacements: { + id: author.id, + lastFirst: derivedFields.lastFirst, + searchName: derivedFields.searchName + }, + transaction + } + ) + } + + if (authors.length < 500) { + return + } + offset += 500 + } +} + +function compareAuthorsForMerge(left, right) { + const leftHasAsin = !!left.asin?.trim() + const rightHasAsin = !!right.asin?.trim() + if (leftHasAsin !== rightHasAsin) return leftHasAsin ? -1 : 1 + + const leftHasDescription = !!left.description?.trim() + const rightHasDescription = !!right.description?.trim() + if (leftHasDescription !== rightHasDescription) return leftHasDescription ? -1 : 1 + + const leftCreatedAt = Number.isFinite(new Date(left.createdAt).getTime()) ? new Date(left.createdAt).getTime() : Number.MAX_SAFE_INTEGER + const rightCreatedAt = Number.isFinite(new Date(right.createdAt).getTime()) ? new Date(right.createdAt).getTime() : Number.MAX_SAFE_INTEGER + if (leftCreatedAt !== rightCreatedAt) return leftCreatedAt - rightCreatedAt + + return String(left.id).localeCompare(String(right.id)) +} + +async function mergeDuplicateAuthors(queryInterface, logger, transaction) { + const authors = await queryInterface.sequelize.query( + `SELECT id, name, asin, description, createdAt, libraryId, searchName + FROM ${AUTHORS_TABLE} + WHERE searchName IS NOT NULL AND searchName != '' + ORDER BY libraryId ASC, searchName ASC, createdAt ASC, id ASC`, + { + type: Sequelize.QueryTypes.SELECT, + transaction + } + ) + + const duplicateGroups = new Map() + for (const author of authors) { + const key = `${author.libraryId}::${author.searchName}` + if (!duplicateGroups.has(key)) duplicateGroups.set(key, []) + duplicateGroups.get(key).push(author) + } + + const groupsToMerge = [...duplicateGroups.values()].filter((authorsInGroup) => authorsInGroup.length > 1) + if (!groupsToMerge.length) { + logger.info(`${loggerPrefix} no duplicate authors found to merge`) + return + } + + logger.info(`${loggerPrefix} merging ${groupsToMerge.length} duplicate author groups`) + + for (const authorsInGroup of groupsToMerge) { + const survivors = [...authorsInGroup].sort(compareAuthorsForMerge) + const survivor = survivors[0] + const duplicateIds = survivors.slice(1).map((author) => author.id) + if (!duplicateIds.length) continue + + logger.info(`${loggerPrefix} merging duplicate authors in library ${survivor.libraryId} for searchName "${survivor.searchName}" into "${survivor.id}"`) + + await queryInterface.sequelize.query( + `UPDATE ${BOOK_AUTHORS_TABLE} + SET authorId = :survivorId + WHERE authorId IN (:duplicateIds)`, + { + replacements: { + survivorId: survivor.id, + duplicateIds + }, + transaction + } + ) + + await queryInterface.sequelize.query( + `DELETE FROM ${AUTHORS_TABLE} + WHERE id IN (:duplicateIds)`, + { + replacements: { + duplicateIds + }, + transaction + } + ) + } +} + +async function cleanupDuplicateBookAuthors(queryInterface, transaction) { + const bookAuthorsTableDescription = await queryInterface.describeTable(BOOK_AUTHORS_TABLE) + if (!bookAuthorsTableDescription?.authorId) return + + await queryInterface.sequelize.query( + `DELETE FROM ${BOOK_AUTHORS_TABLE} + WHERE EXISTS ( + SELECT 1 + FROM ${BOOK_AUTHORS_TABLE} AS duplicateBookAuthors + WHERE duplicateBookAuthors.bookId = ${BOOK_AUTHORS_TABLE}.bookId + AND duplicateBookAuthors.authorId = ${BOOK_AUTHORS_TABLE}.authorId + AND ( + duplicateBookAuthors.createdAt < ${BOOK_AUTHORS_TABLE}.createdAt + OR ( + duplicateBookAuthors.createdAt = ${BOOK_AUTHORS_TABLE}.createdAt + AND duplicateBookAuthors.id < ${BOOK_AUTHORS_TABLE}.id + ) + ) + )`, + { + transaction + } + ) +} + +async function refreshLibraryItemAuthorNames(queryInterface, transaction) { + const libraryItemsTableDescription = await queryInterface.describeTable(LIBRARY_ITEMS_TABLE) + if (!libraryItemsTableDescription?.authorNamesFirstLast || !libraryItemsTableDescription?.authorNamesLastFirst) return + + await queryInterface.sequelize.query( + `UPDATE ${LIBRARY_ITEMS_TABLE} + SET (authorNamesFirstLast, authorNamesLastFirst) = ( + SELECT GROUP_CONCAT(authors.name, ', ' ORDER BY bookAuthors.createdAt ASC), + GROUP_CONCAT(authors.lastFirst, ', ' ORDER BY bookAuthors.createdAt ASC) + FROM authors JOIN bookAuthors ON authors.id = bookAuthors.authorId + WHERE bookAuthors.bookId = ${LIBRARY_ITEMS_TABLE}.mediaId + ) + WHERE mediaType = 'book'`, + { + transaction + } + ) +} + +/** + * This upward migration adds a searchName column to authors and indexes the + * derived fields used for normalized lookup and sorting. + * + * @param {MigrationOptions} options - an object containing the migration context. + * @returns {Promise} - A promise that resolves when the migration is complete. + */ +async function up({ context: { queryInterface, logger } }) { + logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) + + const tableDescription = await queryInterface.describeTable(AUTHORS_TABLE) + + await queryInterface.sequelize.transaction(async (transaction) => { + if (!tableDescription.searchName) { + logger.info(`${loggerPrefix} adding column "searchName" to "${AUTHORS_TABLE}"`) + await queryInterface.addColumn( + AUTHORS_TABLE, + 'searchName', + { + type: Sequelize.DataTypes.STRING + }, + { + transaction + } + ) + } else { + logger.info(`${loggerPrefix} column "searchName" already exists on "${AUTHORS_TABLE}"`) + } + + if (!tableDescription.lastFirst) { + logger.info(`${loggerPrefix} adding column "lastFirst" to "${AUTHORS_TABLE}"`) + await queryInterface.addColumn( + AUTHORS_TABLE, + 'lastFirst', + { + type: Sequelize.DataTypes.STRING + }, + { + transaction + } + ) + } + + await backfillAuthorSearchName(queryInterface, logger, transaction, 0) + await mergeDuplicateAuthors(queryInterface, logger, transaction) + await cleanupDuplicateBookAuthors(queryInterface, transaction) + await refreshLibraryItemAuthorNames(queryInterface, transaction) + + await addIndexIfMissing( + queryInterface, + logger, + AUTHORS_TABLE, + AUTHOR_LAST_FIRST_INDEX, + { + fields: [{ name: 'lastFirst', collate: 'NOCASE' }] + }, + transaction + ) + + await addIndexIfMissing( + queryInterface, + logger, + AUTHORS_TABLE, + AUTHOR_SEARCH_INDEX, + { + fields: [{ name: 'searchName', collate: 'NOCASE' }] + }, + transaction + ) + + await addIndexIfMissing( + queryInterface, + logger, + AUTHORS_TABLE, + UNIQUE_SEARCH_INDEX, + { + fields: ['searchName', 'libraryId'], + unique: true + }, + transaction + ) + }) + + logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`) +} + +/** + * This downward migration removes the searchName column and indexes added by the + * upward migration. + * + * @param {MigrationOptions} options - an object containing the migration context. + * @returns {Promise} - A promise that resolves when the migration is complete. + */ +async function down({ context: { queryInterface, logger } }) { + logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`) + + await removeIndexIfPresent(queryInterface, logger, AUTHORS_TABLE, UNIQUE_SEARCH_INDEX) + await removeIndexIfPresent(queryInterface, logger, AUTHORS_TABLE, AUTHOR_SEARCH_INDEX) + await removeIndexIfPresent(queryInterface, logger, AUTHORS_TABLE, AUTHOR_LAST_FIRST_INDEX) + + const tableDescription = await queryInterface.describeTable(AUTHORS_TABLE) + if (tableDescription.searchName) { + logger.info(`${loggerPrefix} removing column "searchName" from "${AUTHORS_TABLE}"`) + await queryInterface.removeColumn(AUTHORS_TABLE, 'searchName') + } else { + logger.info(`${loggerPrefix} column "searchName" does not exist on "${AUTHORS_TABLE}"`) + } + + logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`) +} + +module.exports = { up, down } diff --git a/test/server/migrations/v2.34.0-add-author-search-name.test.js b/test/server/migrations/v2.34.0-add-author-search-name.test.js new file mode 100644 index 000000000..0fa6578ef --- /dev/null +++ b/test/server/migrations/v2.34.0-add-author-search-name.test.js @@ -0,0 +1,255 @@ +const chai = require('chai') +const sinon = require('sinon') +const { expect } = chai + +const { DataTypes, Sequelize } = require('sequelize') +const Logger = require('../../../server/Logger') +const Author = require('../../../server/models/Author') + +const { up, down } = require('../../../server/migrations/v2.34.0-add-author-search-name') + +describe('Migration v2.34.0-add-author-search-name', () => { + let sequelize + let queryInterface + + beforeEach(async () => { + sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + queryInterface = sequelize.getQueryInterface() + sinon.stub(Logger, 'info') + + await queryInterface.createTable('authors', { + id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true }, + name: { type: DataTypes.STRING, allowNull: false }, + lastFirst: { type: DataTypes.STRING, allowNull: true }, + searchName: { type: DataTypes.STRING, allowNull: true }, + asin: { type: DataTypes.STRING, allowNull: true }, + description: { type: DataTypes.TEXT, allowNull: true }, + libraryId: { type: DataTypes.INTEGER, allowNull: false }, + createdAt: { type: DataTypes.DATE, allowNull: true } + }) + + await queryInterface.createTable('bookAuthors', { + id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true }, + bookId: { type: DataTypes.INTEGER, allowNull: false }, + authorId: { type: DataTypes.INTEGER, allowNull: false }, + createdAt: { type: DataTypes.DATE, allowNull: true } + }) + + await queryInterface.createTable('libraryItems', { + id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true }, + mediaId: { type: DataTypes.INTEGER, allowNull: false }, + mediaType: { type: DataTypes.STRING, allowNull: false }, + authorNamesFirstLast: { type: DataTypes.STRING, allowNull: true }, + authorNamesLastFirst: { type: DataTypes.STRING, allowNull: true } + }) + + await queryInterface.bulkInsert('authors', [ + { id: 1, name: 'J.R.R. Tolkein', lastFirst: 'Tolkein, J. R. R.', asin: null, description: null, libraryId: 1, createdAt: '2020-01-01T00:00:00.000Z' }, + { id: 2, name: 'JRR Tolkein', lastFirst: 'Tolkein, JRR', asin: 'ASIN-1', description: null, libraryId: 1, createdAt: '2021-01-01T00:00:00.000Z' }, + { id: 3, name: 'John Smith', lastFirst: 'Smith, John', asin: null, description: 'Author bio', libraryId: 1, createdAt: '2020-01-02T00:00:00.000Z' }, + { id: 4, name: 'John Smith', lastFirst: 'Smith, John', asin: null, description: null, libraryId: 1, createdAt: '2019-01-01T00:00:00.000Z' }, + { id: 5, name: 'Anna Lee', lastFirst: 'Lee, Anna', asin: null, description: null, libraryId: 1, createdAt: '2022-01-01T00:00:00.000Z' }, + { id: 6, name: 'Anna-Lee', lastFirst: 'Lee, Anna', asin: null, description: null, libraryId: 1, createdAt: '2018-01-01T00:00:00.000Z' }, + { id: 7, name: 'JRR Tolkein', lastFirst: 'Tolkein, JRR', asin: null, description: null, libraryId: 2, createdAt: '2021-06-01T00:00:00.000Z' }, + { id: 8, name: 'Agatha Christie', lastFirst: 'Christie, Agatha', asin: null, description: null, libraryId: 2, createdAt: '2020-06-01T00:00:00.000Z' } + ]) + + await queryInterface.bulkInsert('bookAuthors', [ + { id: 1, bookId: 101, authorId: 1, createdAt: '2020-02-01T00:00:00.000Z' }, + { id: 2, bookId: 101, authorId: 2, createdAt: '2020-03-01T00:00:00.000Z' }, + { id: 3, bookId: 102, authorId: 3, createdAt: '2020-02-01T00:00:00.000Z' }, + { id: 4, bookId: 102, authorId: 4, createdAt: '2020-03-01T00:00:00.000Z' }, + { id: 5, bookId: 103, authorId: 5, createdAt: '2020-02-01T00:00:00.000Z' }, + { id: 6, bookId: 103, authorId: 6, createdAt: '2020-03-01T00:00:00.000Z' }, + { id: 7, bookId: 104, authorId: 7, createdAt: '2020-02-01T00:00:00.000Z' }, + { id: 8, bookId: 104, authorId: 8, createdAt: '2020-03-01T00:00:00.000Z' } + ]) + + await queryInterface.bulkInsert('libraryItems', [ + { id: 1, mediaId: 101, mediaType: 'book', authorNamesFirstLast: 'stale', authorNamesLastFirst: 'stale' }, + { id: 2, mediaId: 102, mediaType: 'book', authorNamesFirstLast: 'stale', authorNamesLastFirst: 'stale' }, + { id: 3, mediaId: 103, mediaType: 'book', authorNamesFirstLast: 'stale', authorNamesLastFirst: 'stale' }, + { id: 4, mediaId: 104, mediaType: 'book', authorNamesFirstLast: 'stale', authorNamesLastFirst: 'stale' } + ]) + }) + + afterEach(() => { + sinon.restore() + }) + + describe('up', () => { + it('should backfill derived author fields before adding indexes', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const authors = await queryInterface.sequelize.query('SELECT id, name, lastFirst, searchName, asin, description, libraryId FROM authors ORDER BY id ASC') + expect(authors[0]).to.deep.equal([ + { + id: 2, + name: 'JRR Tolkein', + lastFirst: 'Tolkein, JRR', + searchName: 'jrrtolkein', + asin: 'ASIN-1', + description: null, + libraryId: 1 + }, + { + id: 3, + name: 'John Smith', + lastFirst: 'Smith, John', + searchName: 'johnsmith', + asin: null, + description: 'Author bio', + libraryId: 1 + }, + { + id: 6, + name: 'Anna-Lee', + lastFirst: 'Anna-Lee', + searchName: 'annalee', + asin: null, + description: null, + libraryId: 1 + }, + { + id: 7, + name: 'JRR Tolkein', + lastFirst: 'Tolkein, JRR', + searchName: 'jrrtolkein', + asin: null, + description: null, + libraryId: 2 + }, + { + id: 8, + name: 'Agatha Christie', + lastFirst: 'Christie, Agatha', + searchName: 'agathachristie', + asin: null, + description: null, + libraryId: 2 + } + ]) + }) + + it('should merge duplicate authors per library and remap bookAuthors', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const authors = await queryInterface.sequelize.query('SELECT id, name, lastFirst, searchName, asin, description, libraryId FROM authors ORDER BY id ASC') + expect(authors[0]).to.deep.equal([ + { + id: 2, + name: 'JRR Tolkein', + lastFirst: 'Tolkein, JRR', + searchName: 'jrrtolkein', + asin: 'ASIN-1', + description: null, + libraryId: 1 + }, + { + id: 3, + name: 'John Smith', + lastFirst: 'Smith, John', + searchName: 'johnsmith', + asin: null, + description: 'Author bio', + libraryId: 1 + }, + { + id: 6, + name: 'Anna-Lee', + lastFirst: 'Anna-Lee', + searchName: 'annalee', + asin: null, + description: null, + libraryId: 1 + }, + { + id: 7, + name: 'JRR Tolkein', + lastFirst: 'Tolkein, JRR', + searchName: 'jrrtolkein', + asin: null, + description: null, + libraryId: 2 + }, + { + id: 8, + name: 'Agatha Christie', + lastFirst: 'Christie, Agatha', + searchName: 'agathachristie', + asin: null, + description: null, + libraryId: 2 + } + ]) + + const bookAuthors = await queryInterface.sequelize.query('SELECT bookId, authorId FROM bookAuthors ORDER BY bookId ASC, authorId ASC') + expect(bookAuthors[0]).to.deep.equal([ + { bookId: 101, authorId: 2 }, + { bookId: 102, authorId: 3 }, + { bookId: 103, authorId: 6 }, + { bookId: 104, authorId: 7 }, + { bookId: 104, authorId: 8 } + ]) + + const libraryItems = await queryInterface.sequelize.query('SELECT mediaId, authorNamesFirstLast, authorNamesLastFirst FROM libraryItems ORDER BY mediaId ASC') + expect(libraryItems[0]).to.deep.equal([ + { mediaId: 101, authorNamesFirstLast: 'JRR Tolkein', authorNamesLastFirst: 'Tolkein, JRR' }, + { mediaId: 102, authorNamesFirstLast: 'John Smith', authorNamesLastFirst: 'Smith, John' }, + { mediaId: 103, authorNamesFirstLast: 'Anna-Lee', authorNamesLastFirst: 'Anna-Lee' }, + { mediaId: 104, authorNamesFirstLast: 'JRR Tolkein, Agatha Christie', authorNamesLastFirst: 'Tolkein, JRR, Christie, Agatha' } + ]) + }) + + it('should create indexes after the merge completes', async () => { + await up({ context: { queryInterface, logger: Logger } }) + + const [[{ count: lastFirstCount }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='author_last_first'`) + expect(lastFirstCount).to.equal(1) + + const [[{ count: searchNameCount }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='author_search_name'`) + expect(searchNameCount).to.equal(1) + + const [[{ count: uniqueCount }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='unique_author_search_name_per_library'`) + expect(uniqueCount).to.equal(1) + + const [[{ count: duplicateCount }]] = await queryInterface.sequelize.query( + `SELECT COUNT(*) as count FROM authors WHERE libraryId = :libraryId AND searchName = :searchName`, + { + replacements: { + libraryId: 1, + searchName: Author.normalizeSearchName('JRR Tolkein') + } + } + ) + expect(Number(duplicateCount)).to.equal(1) + }) + + it('should be idempotent', async () => { + await up({ context: { queryInterface, logger: Logger } }) + await up({ context: { queryInterface, logger: Logger } }) + + const tableDescription = await queryInterface.describeTable('authors') + expect(tableDescription.searchName).to.exist + + const [[{ count: uniqueCount }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='unique_author_search_name_per_library'`) + expect(uniqueCount).to.equal(1) + }) + }) + + describe('down', () => { + it('should remove searchName and its indexes', async () => { + await up({ context: { queryInterface, logger: Logger } }) + await down({ context: { queryInterface, logger: Logger } }) + + const tableDescription = await queryInterface.describeTable('authors') + expect(tableDescription.searchName).to.not.exist + + const [[{ count: searchNameCount }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='author_search_name'`) + expect(searchNameCount).to.equal(0) + + const [[{ count: uniqueCount }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='unique_author_search_name_per_library'`) + expect(uniqueCount).to.equal(0) + }) + }) +}) From f16caab7819099ffc1cd629f4d354c77fe948b2f Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Wed, 13 May 2026 14:51:02 -0700 Subject: [PATCH 06/12] Update author filter to use normalized search name --- server/utils/queries/authorFilters.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/server/utils/queries/authorFilters.js b/server/utils/queries/authorFilters.js index 3d6bc7bd6..63edd02ce 100644 --- a/server/utils/queries/authorFilters.js +++ b/server/utils/queries/authorFilters.js @@ -1,5 +1,6 @@ const Sequelize = require('sequelize') const Database = require('../../Database') +const Author = require('../../models/Author') module.exports = { /** @@ -60,10 +61,19 @@ module.exports = { * @returns {Promise} oldAuthor with numBooks */ async search(libraryId, query, limit, offset) { - const matchAuthor = query.matchExpression('name') + const normalizedQuery = Author.normalizeSearchName(query.query) + if (!normalizedQuery) { + return [] + } + const authors = await Database.authorModel.findAll({ where: { - [Sequelize.Op.and]: [Sequelize.literal(matchAuthor), { libraryId }] + [Sequelize.Op.and]: [ + Sequelize.where(Sequelize.col('searchName'), { + [Sequelize.Op.substring]: normalizedQuery + }), + { libraryId } + ] }, attributes: { include: [[Sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 'numBooks']] From a3978ffcba102adad2462710d42a3e69a532e38d Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Wed, 13 May 2026 14:54:30 -0700 Subject: [PATCH 07/12] Initial author and filter tests created --- test/server/models/Author.test.js | 76 +++++++++++++++ .../utils/queries/authorFilters.test.js | 96 +++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 test/server/models/Author.test.js create mode 100644 test/server/utils/queries/authorFilters.test.js diff --git a/test/server/models/Author.test.js b/test/server/models/Author.test.js new file mode 100644 index 000000000..0a3c4435d --- /dev/null +++ b/test/server/models/Author.test.js @@ -0,0 +1,76 @@ +const chai = require('chai') +const sinon = require('sinon') +const { expect } = chai + +const { Sequelize } = require('sequelize') +const Database = require('../../../server/Database') +const Author = require('../../../server/models/Author') +const Library = require('../../../server/models/Library') + +describe('Author model', () => { + let sequelize + + beforeEach(async () => { + sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + + Library.init(sequelize) + Author.init(sequelize) + + await sequelize.sync({ force: true }) + await Library.create({ + id: '00000000-0000-0000-0000-000000000001', + name: 'Test Library', + displayOrder: 1, + mediaType: 'book' + }) + }) + + afterEach(async () => { + sinon.restore() + Database.sequelize = null + await sequelize?.close() + }) + + describe('findOrCreateByNameAndLibrary', () => { + it('returns an error when the normalized author name is empty', async () => { + const result = await Author.findOrCreateByNameAndLibrary(' ', '00000000-0000-0000-0000-000000000001') + + expect(result.author).to.equal(null) + expect(result.created).to.equal(false) + expect(result.error).to.equal(undefined) + + const count = await Author.count() + expect(count).to.equal(0) + }) + }) + + describe('Database.rebuildAuthorRows', () => { + it('rebuilds stale derived author fields during initialization', async () => { + const db = Database + db.sequelize = sequelize + + await sequelize.getQueryInterface().bulkInsert('authors', [ + { + id: '00000000-0000-0000-0000-000000000010', + name: 'Gabriel García Márquez', + lastFirst: 'wrong', + searchName: 'wrong', + libraryId: '00000000-0000-0000-0000-000000000001', + createdAt: '2025-01-01 00:00:00.000 +00:00', + updatedAt: '2025-01-01 00:00:00.000 +00:00' + } + ]) + + await db.rebuildAuthorRows() + + const [authors] = await sequelize.query('SELECT name, lastFirst, searchName FROM authors') + expect(authors).to.deep.equal([ + { + name: 'Gabriel García Márquez', + lastFirst: 'Márquez, Gabriel García', + searchName: 'gabrielgarciamarquez' + } + ]) + }) + }) +}) diff --git a/test/server/utils/queries/authorFilters.test.js b/test/server/utils/queries/authorFilters.test.js new file mode 100644 index 000000000..7e2b518ad --- /dev/null +++ b/test/server/utils/queries/authorFilters.test.js @@ -0,0 +1,96 @@ +const chai = require('chai') +const { expect } = chai +const { Sequelize } = require('sequelize') + +const Database = require('../../../../server/Database') +const Author = require('../../../../server/models/Author') +const Library = require('../../../../server/models/Library') +const authorFilters = require('../../../../server/utils/queries/authorFilters') + +describe('authorFilters', () => { + let sequelize + + beforeEach(async () => { + sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + + Library.init(sequelize) + Author.init(sequelize) + + await sequelize.sync({ force: true }) + await sequelize.getQueryInterface().createTable('bookAuthors', { + id: { type: Sequelize.DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true }, + bookId: { type: Sequelize.DataTypes.INTEGER, allowNull: false }, + authorId: { type: Sequelize.DataTypes.INTEGER, allowNull: false }, + createdAt: { type: Sequelize.DataTypes.DATE, allowNull: true } + }) + + Database.sequelize = sequelize + Database.authorModel = Author + + await Library.create({ + id: '00000000-0000-0000-0000-000000000001', + name: 'Test Library', + displayOrder: 1, + mediaType: 'book' + }) + + await Author.bulkCreate([ + { + id: '00000000-0000-0000-0000-000000000010', + name: 'J.R.R. Tolkein', + lastFirst: 'Tolkein, J. R. R.', + searchName: 'jrrtolkein', + libraryId: '00000000-0000-0000-0000-000000000001', + createdAt: '2025-01-01 00:00:00.000 +00:00', + updatedAt: '2025-01-01 00:00:00.000 +00:00' + }, + { + id: '00000000-0000-0000-0000-000000000011', + name: 'Agatha Christie', + lastFirst: 'Christie, Agatha', + searchName: 'agathachristie', + libraryId: '00000000-0000-0000-0000-000000000001', + createdAt: '2025-01-01 00:00:00.000 +00:00', + updatedAt: '2025-01-01 00:00:00.000 +00:00' + } + ]) + + await sequelize.getQueryInterface().bulkInsert('bookAuthors', [ + { + id: 1, + bookId: 1, + authorId: '00000000-0000-0000-0000-000000000010', + createdAt: '2025-01-01 00:00:00.000 +00:00' + }, + { + id: 2, + bookId: 2, + authorId: '00000000-0000-0000-0000-000000000011', + createdAt: '2025-01-01 00:00:00.000 +00:00' + } + ]) + }) + + afterEach(async () => { + await sequelize?.close() + }) + + it('matches authors by normalized searchName as well as display name', async () => { + const query = await Database.createTextSearchQuery('jrr') + const results = await authorFilters.search('00000000-0000-0000-0000-000000000001', query, 10, 0) + + expect(results).to.deep.equal([ + { + id: '00000000-0000-0000-0000-000000000010', + asin: null, + name: 'J.R.R. Tolkein', + description: null, + imagePath: null, + libraryId: '00000000-0000-0000-0000-000000000001', + addedAt: 1735689600000, + updatedAt: 1735689600000, + numBooks: 1 + } + ]) + }) +}) From a60b2120193d94eb2180687e401950bd4dab6705 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Wed, 13 May 2026 15:34:51 -0700 Subject: [PATCH 08/12] Create helper function to regenerate rows in tables faster --- server/Database.js | 52 +++++++++++++++++++++++++++++------------ server/models/Author.js | 19 +++++++++++++++ 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/server/Database.js b/server/Database.js index 316880191..4259c4668 100644 --- a/server/Database.js +++ b/server/Database.js @@ -211,34 +211,56 @@ class Database { } /** - * Rebuild all author rows so derived fields stay in sync with the model logic. + * Rebuild rows for a model so hooks can recompute derived fields. + * + * @param {import('sequelize').ModelStatic} model + * @param {{ + * label?: string, + * pageSize?: number, + * attributes?: string[], + * order?: import('sequelize').Order, + * prepareRow?: (row: any) => void | Promise + * }} [options] */ - async rebuildAuthorRows() { - const pageSize = 500 + async rebuildComputedTableRows(model, options = {}) { + const { label = model?.name || 'rows', pageSize = 500, attributes = ['id'], order = [['id', 'ASC']], prepareRow = async () => {} } = options + let offset = 0 + let totalChange = 0 while (true) { - const authors = await this.authorModel.findAll({ - attributes: ['id', 'name', 'libraryId'], - order: [['id', 'ASC']], + const rows = await model.findAll({ + attributes, + order, limit: pageSize, offset }) - if (!authors.length) break + if (!rows.length) break - for (const authorRef of authors) { - const author = await this.authorModel.findByPk(authorRef.id) - if (!author) continue - author.changed('name', true) - await author.save({ silent: true }) + for (const row of rows) { + await prepareRow(row) + if (!row.changed()) continue + totalChange++ + await row.save({ silent: true }) } - offset += authors.length - if (authors.length < pageSize) break + offset += rows.length + if (rows.length < pageSize) break } - Logger.debug(`Sanitized ${offset} author rows`) + Logger.debug(`${totalChange} out of ${offset} ${label} rows required recalculation of derived columns`) + } + + /** + * Rebuild all author rows so derived fields stay in sync with the model logic. + */ + async rebuildAuthorRows() { + await this.rebuildComputedTableRows(this.authorModel, { + label: 'author', + attributes: ['id', 'name', 'lastFirst', 'searchName', 'libraryId'], + prepareRow: (author) => this.authorModel.isDerivedFieldChange(author) + }) } /** diff --git a/server/models/Author.js b/server/models/Author.js index aedeb8777..4d680f27c 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -66,6 +66,25 @@ class Author extends Model { return this.normalizeSearchName(leftName) === this.normalizeSearchName(rightName) } + static isDerivedFieldChange(author) { + const derivedFields = this.buildAuthorDerivedFields(author.name) + let changed = false + + if (author.lastFirst !== derivedFields.lastFirst) { + author.setDataValue('lastFirst', derivedFields.lastFirst) + author.changed('lastFirst', true) + changed = true + } + + if (author.searchName !== derivedFields.searchName) { + author.setDataValue('searchName', derivedFields.searchName) + author.changed('searchName', true) + changed = true + } + + return changed + } + /** * Check if author exists * @param {string} authorId From 21bede399d72e6a86291c78d6984ee85ca6da62a Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Wed, 13 May 2026 15:35:27 -0700 Subject: [PATCH 09/12] Update function name to be more descriptive --- server/Database.js | 2 +- server/models/Author.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/Database.js b/server/Database.js index 4259c4668..9237c1cf9 100644 --- a/server/Database.js +++ b/server/Database.js @@ -259,7 +259,7 @@ class Database { await this.rebuildComputedTableRows(this.authorModel, { label: 'author', attributes: ['id', 'name', 'lastFirst', 'searchName', 'libraryId'], - prepareRow: (author) => this.authorModel.isDerivedFieldChange(author) + prepareRow: (author) => this.authorModel.hasDerivedFieldChange(author) }) } diff --git a/server/models/Author.js b/server/models/Author.js index 4d680f27c..07d36d3ed 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -66,7 +66,7 @@ class Author extends Model { return this.normalizeSearchName(leftName) === this.normalizeSearchName(rightName) } - static isDerivedFieldChange(author) { + static hasDerivedFieldChange(author) { const derivedFields = this.buildAuthorDerivedFields(author.name) let changed = false From c46cb4769e99ded2a95e7fa56f87b6f30119fff3 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Wed, 13 May 2026 15:40:36 -0700 Subject: [PATCH 10/12] Update author model function comments --- server/models/Author.js | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/server/models/Author.js b/server/models/Author.js index 07d36d3ed..9708338cd 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -37,6 +37,11 @@ class Author extends Model { return parseNameString.nameToLastFirst(name) } + /** + * Remove all punctionation, diacritics, and whitespace and convert to lowercase for searching and matching + * @param {string} name + * @returns {string} + */ static normalizeSearchName(name) { if (!name?.trim()) return null return name @@ -47,6 +52,11 @@ class Author extends Model { .trim() } + /** + * Calculate derived fields. Returns null if name is empty after normalization + * @param {string} name + * @returns { lastFirst: string?, searchName: string? } + */ static buildAuthorDerivedFields(name) { const searchName = this.normalizeSearchName(name) if (!searchName) { @@ -62,10 +72,21 @@ class Author extends Model { } } + /** + * Check if two author names match after normalization + * @param {string} leftName + * @param {string} rightName + * @returns {boolean} + */ static isAuthorNameMatch(leftName, rightName) { return this.normalizeSearchName(leftName) === this.normalizeSearchName(rightName) } + /** + * Check if any derived fields would change to reduce unnecessary database writes + * @param {Author} author + * @returns + */ static hasDerivedFieldChange(author) { const derivedFields = this.buildAuthorDerivedFields(author.name) let changed = false @@ -96,8 +117,6 @@ class Author extends Model { /** * Get author by name and libraryId. name case insensitive - * TODO: Look for authors ignoring punctuation - * * @param {string} authorName * @param {string} libraryId * @returns {Promise} @@ -158,7 +177,7 @@ class Author extends Model { } /** - * + * Ensure duplicate authors are not created for the same library using the normalized name * @param {string} name * @param {string} libraryId * @returns {Promise<{ author: Author, created: boolean }>} From 68eb48bef863f1064655377ad9fe00ab2162e163 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Wed, 13 May 2026 15:49:24 -0700 Subject: [PATCH 11/12] Replace findOrCreate with findCreateFind to remove transaction requirement --- server/models/Author.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models/Author.js b/server/models/Author.js index 9708338cd..8ca45b63a 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -188,7 +188,7 @@ class Author extends Model { return { author: null, created: false } } - const [author, created] = await this.findOrCreate({ + const [author, created] = await this.findCreateFind({ where: { searchName, libraryId From 58501403695b38f134cc295a3fb458f1ea18ccc9 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Wed, 13 May 2026 17:09:46 -0700 Subject: [PATCH 12/12] Move Author import to upgrade function to prevent potential downgrade issue --- server/migrations/v2.34.0-add-author-search-name.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/migrations/v2.34.0-add-author-search-name.js b/server/migrations/v2.34.0-add-author-search-name.js index ddac8d645..1b691478a 100644 --- a/server/migrations/v2.34.0-add-author-search-name.js +++ b/server/migrations/v2.34.0-add-author-search-name.js @@ -50,7 +50,7 @@ async function removeIndexIfPresent(queryInterface, logger, tableName, indexName await queryInterface.removeIndex(tableName, indexName) } -async function backfillAuthorSearchName(queryInterface, logger, transaction, offset = 0) { +async function backfillAuthorSearchName(queryInterface, logger, transaction, Author, offset = 0) { while (true) { const authors = await queryInterface.sequelize.query(`SELECT id, name FROM ${AUTHORS_TABLE} ORDER BY id ASC LIMIT :limit OFFSET :offset`, { replacements: { limit: 500, offset }, @@ -217,6 +217,7 @@ async function refreshLibraryItemAuthorNames(queryInterface, transaction) { async function up({ context: { queryInterface, logger } }) { logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`) + const Author = require('../models/Author') const tableDescription = await queryInterface.describeTable(AUTHORS_TABLE) await queryInterface.sequelize.transaction(async (transaction) => { @@ -250,7 +251,7 @@ async function up({ context: { queryInterface, logger } }) { ) } - await backfillAuthorSearchName(queryInterface, logger, transaction, 0) + await backfillAuthorSearchName(queryInterface, logger, transaction, Author, 0) await mergeDuplicateAuthors(queryInterface, logger, transaction) await cleanupDuplicateBookAuthors(queryInterface, transaction) await refreshLibraryItemAuthorNames(queryInterface, transaction)