diff --git a/server/Database.js b/server/Database.js index 3d48980c1..3cbf32819 100644 --- a/server/Database.js +++ b/server/Database.js @@ -412,6 +412,11 @@ class Database { require('./models/CustomMetadataProvider').init(this.sequelize) require('./models/MediaItemShare').init(this.sequelize) + if (this.isPostgresDialect() && !force && !this.isNew) { + Logger.info('[Database] Skipping sequelize.sync for existing postgres schema') + return Promise.resolve() + } + return this.sequelize.sync({ force, alter: false }) } diff --git a/server/models/Book.js b/server/models/Book.js index 08982bb03..ac71d207e 100644 --- a/server/models/Book.js +++ b/server/models/Book.js @@ -148,7 +148,7 @@ class Book extends Model { }, title: DataTypes.STRING, titleIgnorePrefix: DataTypes.STRING, - subtitle: DataTypes.STRING, + subtitle: DataTypes.TEXT, publishedYear: DataTypes.STRING, publishedDate: DataTypes.STRING, publisher: DataTypes.STRING, diff --git a/server/models/PlaybackSession.js b/server/models/PlaybackSession.js index 196fbda6c..47526d5f6 100644 --- a/server/models/PlaybackSession.js +++ b/server/models/PlaybackSession.js @@ -190,7 +190,7 @@ class PlaybackSession extends Model { currentTime: DataTypes.FLOAT, serverVersion: DataTypes.STRING, coverPath: DataTypes.STRING, - timeListening: DataTypes.INTEGER, + timeListening: DataTypes.FLOAT, mediaMetadata: DataTypes.JSON, date: DataTypes.STRING, dayOfWeek: DataTypes.STRING, diff --git a/server/scripts/migrateSqliteToPostgres.js b/server/scripts/migrateSqliteToPostgres.js index e8b72881c..43dc439da 100644 --- a/server/scripts/migrateSqliteToPostgres.js +++ b/server/scripts/migrateSqliteToPostgres.js @@ -9,11 +9,6 @@ const PG_SCHEMA = process.env.PG_SCHEMA || 'public' const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500) const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true' -if (!DATABASE_URL) { - console.error('[migrate] DATABASE_URL is required') - process.exit(1) -} - const preferredOrder = [ 'migrationsMeta', 'SequelizeMeta', @@ -75,6 +70,120 @@ function sqliteGet(db, sql, params = []) { }) } +async function findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) { + const issues = [] + + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const pgColumns = pgColumnsByTable.get(postgresTable) + if (!pgColumns) continue + + for (const column of pgColumns.values()) { + if (column.data_type !== 'character varying' || !column.character_maximum_length) continue + + const sqliteColumn = column.column_name + const maxLength = Number(column.character_maximum_length) + const quotedTable = quoteIdent(sqliteTable) + const quotedColumn = quoteIdent(sqliteColumn) + + try { + const maxLengthRow = await sqliteGet( + sqliteDb, + `SELECT MAX(LENGTH(${quotedColumn})) AS maxLength FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL` + ) + const actualMaxLength = Number(maxLengthRow?.maxLength || 0) + if (actualMaxLength <= maxLength) continue + + const overCountRow = await sqliteGet( + sqliteDb, + `SELECT COUNT(*) AS count FROM ${quotedTable} WHERE LENGTH(${quotedColumn}) > ?`, + [maxLength] + ) + + issues.push({ + sqliteTable, + sqliteColumn, + postgresTable, + postgresColumn: column.column_name, + maxLength, + actualMaxLength, + overCount: Number(overCountRow?.count || 0) + }) + } catch (error) { + // Ignore columns missing in sqlite source table + } + } + } + + return issues +} + +function isIntegerCompatible(value) { + if (value === null || value === undefined) return true + + if (typeof value === 'number') { + return Number.isFinite(value) && Number.isInteger(value) + } + + if (typeof value === 'string') { + const trimmed = value.trim() + if (!trimmed) return false + if (!/^-?\d+$/.test(trimmed)) return false + const parsed = Number(trimmed) + return Number.isFinite(parsed) + } + + return false +} + +async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) { + const issues = [] + + for (const { sqliteTable, postgresTable } of tablesToMigrate) { + const pgColumns = pgColumnsByTable.get(postgresTable) + if (!pgColumns) continue + + for (const column of pgColumns.values()) { + const dataType = column.data_type + if (dataType !== 'smallint' && dataType !== 'integer' && dataType !== 'bigint') continue + + const sqliteColumn = column.column_name + const quotedTable = quoteIdent(sqliteTable) + const quotedColumn = quoteIdent(sqliteColumn) + + let rows + try { + rows = await sqliteAll(sqliteDb, `SELECT ${quotedColumn} AS value FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`) + } catch (error) { + // Ignore columns missing in sqlite source table + continue + } + + let badCount = 0 + let sampleValue = null + for (const row of rows) { + if (!isIntegerCompatible(row.value)) { + badCount += 1 + if (sampleValue === null) sampleValue = row.value + } + } + + if (badCount > 0) { + issues.push({ + sqliteTable, + sqliteColumn, + postgresTable, + postgresColumn: column.column_name, + postgresType: dataType, + badCount, + sampleValue + }) + } + } + } + + return issues +} + function normalizeBoolean(value) { if (value === null || value === undefined) return null if (typeof value === 'boolean') return value @@ -88,7 +197,7 @@ function normalizeBoolean(value) { function normalizeJson(value) { if (value === null || value === undefined || value === '') return null - if (typeof value === 'object') return value + if (typeof value === 'object') return JSON.stringify(value) const textValue = String(value) @@ -96,14 +205,14 @@ function normalizeJson(value) { const parsed = JSON.parse(textValue) if (typeof parsed === 'string') { try { - return JSON.parse(parsed) + return JSON.stringify(JSON.parse(parsed)) } catch (error) { - return parsed + return JSON.stringify(parsed) } } - return parsed + return JSON.stringify(parsed) } catch (error) { - // Keep non-JSON payloads as JSON string values so inserts remain valid. + // Keep non-JSON payloads as JSON string values so inserts remain valid JSON. return JSON.stringify(textValue) } } @@ -121,10 +230,45 @@ function convertValue(value, pgColumn) { return normalizeJson(value) } + if ((dataType === 'smallint' || dataType === 'integer' || dataType === 'bigint') && value !== null && value !== undefined) { + if (typeof value === 'number') return value + if (typeof value === 'string' && /^-?\d+$/.test(value.trim())) { + return Number(value) + } + } + return value } +function getOverlongColumns(row, insertColumns) { + const overlong = [] + + for (const column of insertColumns) { + const maxLength = column.metadata.character_maximum_length + if (!maxLength) continue + + const value = row[column.sqliteColumn] + if (value === null || value === undefined) continue + + const length = String(value).length + if (length > maxLength) { + overlong.push({ + sqliteColumn: column.sqliteColumn, + postgresColumn: column.postgresColumn, + maxLength, + actualLength: length + }) + } + } + + return overlong +} + async function main() { + if (!DATABASE_URL) { + throw new Error('DATABASE_URL is required') + } + console.log(`[migrate] sqlite source: ${SQLITE_PATH}`) console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`) console.log(`[migrate] dry run: ${DRY_RUN}`) @@ -176,13 +320,38 @@ async function main() { const pgColumnsByTable = new Map() for (const { postgresTable } of tablesToMigrate) { const columnsResult = await pg.query( - `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, + `SELECT column_name, data_type, udt_name, character_maximum_length FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, [PG_SCHEMA, postgresTable] ) const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column])) pgColumnsByTable.set(postgresTable, columnMap) } + const overlongVarcharIssues = await findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) + const integerTypeIssues = await findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) + + if (overlongVarcharIssues.length) { + console.error('[migrate] overlong source values detected for varchar columns:') + overlongVarcharIssues.forEach((issue) => { + console.error( + `[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` + + `(max=${issue.maxLength}, actualMax=${issue.actualMaxLength}, overRows=${issue.overCount})` + ) + }) + throw new Error('Migration aborted to prevent truncation/data loss. Widen target column types first.') + } + + if (integerTypeIssues.length) { + console.error('[migrate] non-integer source values detected for integer columns:') + integerTypeIssues.forEach((issue) => { + console.error( + `[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` + + `(type=${issue.postgresType}, badRows=${issue.badCount}, sample=${JSON.stringify(issue.sampleValue)})` + ) + }) + throw new Error('Migration aborted to prevent numeric precision loss. Widen target numeric column types first.') + } + if (!DRY_RUN) { await pg.query('BEGIN') await pg.query('SET session_replication_role = replica') @@ -222,16 +391,49 @@ async function main() { let paramIndex = 1 for (const row of batchRows) { - const placeholders = [] - for (const column of insertColumns) { - params.push(convertValue(row[column.sqliteColumn], column.metadata)) - placeholders.push(`$${paramIndex++}`) - } + const placeholders = [] + for (const column of insertColumns) { + params.push(convertValue(row[column.sqliteColumn], column.metadata)) + placeholders.push(`$${paramIndex++}`) + } valuesSql.push(`(${placeholders.join(', ')})`) } const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}` - await pg.query(insertSql, params) + + try { + await pg.query('SAVEPOINT migrate_batch') + await pg.query(insertSql, params) + await pg.query('RELEASE SAVEPOINT migrate_batch') + } catch (error) { + await pg.query('ROLLBACK TO SAVEPOINT migrate_batch') + console.error(`[migrate] batch insert failed for ${sqliteTable} (offset=${offset}, size=${batchRows.length}): ${error.message}`) + + for (let rowIndex = 0; rowIndex < batchRows.length; rowIndex++) { + const row = batchRows[rowIndex] + const singleRowParams = insertColumns.map((column) => convertValue(row[column.sqliteColumn], column.metadata)) + const singleRowInsertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES (${singleRowParams.map((_, index) => `$${index + 1}`).join(', ')})` + + try { + await pg.query('SAVEPOINT migrate_row') + await pg.query(singleRowInsertSql, singleRowParams) + await pg.query('RELEASE SAVEPOINT migrate_row') + } catch (rowError) { + await pg.query('ROLLBACK TO SAVEPOINT migrate_row') + const overlongColumns = getOverlongColumns(row, insertColumns) + if (overlongColumns.length) { + overlongColumns.forEach((column) => { + console.error( + `[migrate] overlong value in ${sqliteTable}.${column.sqliteColumn} -> ${postgresTable}.${column.postgresColumn} ` + + `(length=${column.actualLength}, max=${column.maxLength})` + ) + }) + } + + throw rowError + } + } + } } } @@ -271,7 +473,18 @@ async function main() { } } -main().catch((error) => { - console.error('[migrate] failed:', error) - process.exit(1) -}) +if (require.main === module) { + main().catch((error) => { + console.error('[migrate] failed:', error) + process.exit(1) + }) +} + +module.exports = { + normalizeJson, + isIntegerCompatible, + convertValue, + findOverlongVarcharValues, + findIntegerTypeIssues, + quoteIdent +} diff --git a/test/server/scripts/migrateSqliteToPostgres.test.js b/test/server/scripts/migrateSqliteToPostgres.test.js new file mode 100644 index 000000000..9b218079e --- /dev/null +++ b/test/server/scripts/migrateSqliteToPostgres.test.js @@ -0,0 +1,131 @@ +const { expect } = require('chai') +const sqlite3 = require('sqlite3') + +const { + normalizeJson, + isIntegerCompatible, + findOverlongVarcharValues, + findIntegerTypeIssues +} = require('../../../server/scripts/migrateSqliteToPostgres') + +function openMemoryDb() { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(':memory:', (error) => { + if (error) return reject(error) + resolve(db) + }) + }) +} + +function run(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.run(sql, params, (error) => { + if (error) return reject(error) + resolve() + }) + }) +} + +function close(db) { + return new Promise((resolve, reject) => { + db.close((error) => { + if (error) return reject(error) + resolve() + }) + }) +} + +describe('migrateSqliteToPostgres script helpers', () => { + it('should keep malformed JSON payloads insertable by returning valid JSON text', () => { + const malformed = '"{"x",1}"' + const normalized = normalizeJson(malformed) + + expect(() => JSON.parse(normalized)).to.not.throw() + expect(JSON.parse(normalized)).to.equal('"{"x",1}"') + }) + + it('should detect overlong varchar values before migration', async () => { + const db = await openMemoryDb() + + try { + await run(db, 'CREATE TABLE books (subtitle TEXT)') + await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['x'.repeat(300)]) + await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['ok']) + + const tablesToMigrate = [{ sqliteTable: 'books', postgresTable: 'books' }] + const pgColumnsByTable = new Map([ + [ + 'books', + new Map([ + [ + 'subtitle', + { + column_name: 'subtitle', + data_type: 'character varying', + character_maximum_length: 255 + } + ] + ]) + ] + ]) + + const issues = await findOverlongVarcharValues(db, tablesToMigrate, pgColumnsByTable) + + expect(issues).to.have.length(1) + expect(issues[0]).to.include({ + sqliteTable: 'books', + sqliteColumn: 'subtitle', + postgresTable: 'books', + postgresColumn: 'subtitle', + maxLength: 255, + actualMaxLength: 300, + overCount: 1 + }) + } finally { + await close(db) + } + }) + + it('should detect non-integer values for integer target columns', async () => { + const db = await openMemoryDb() + + try { + await run(db, 'CREATE TABLE playbackSessions (timeListening REAL)') + await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [25.802536999999997]) + await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [42]) + + const tablesToMigrate = [{ sqliteTable: 'playbackSessions', postgresTable: 'playbackSessions' }] + const pgColumnsByTable = new Map([ + [ + 'playbackSessions', + new Map([ + [ + 'timelistening', + { + column_name: 'timeListening', + data_type: 'integer' + } + ] + ]) + ] + ]) + + const issues = await findIntegerTypeIssues(db, tablesToMigrate, pgColumnsByTable) + + expect(issues).to.have.length(1) + expect(issues[0].sqliteTable).to.equal('playbackSessions') + expect(issues[0].sqliteColumn).to.equal('timeListening') + expect(issues[0].postgresType).to.equal('integer') + expect(issues[0].badCount).to.equal(1) + } finally { + await close(db) + } + }) + + it('should only accept integer-compatible values for integer columns', () => { + expect(isIntegerCompatible(10)).to.equal(true) + expect(isIntegerCompatible('10')).to.equal(true) + expect(isIntegerCompatible(10.5)).to.equal(false) + expect(isIntegerCompatible('10.5')).to.equal(false) + }) +})