prevent lossy postgres migration edge cases

server/Database.js

@@ -412,6 +412,11 @@ class Database {
     require('./models/CustomMetadataProvider').init(this.sequelize)
     require('./models/MediaItemShare').init(this.sequelize)
 
+    if (this.isPostgresDialect() && !force && !this.isNew) {
+      Logger.info('[Database] Skipping sequelize.sync for existing postgres schema')
+      return Promise.resolve()
+    }
+
     return this.sequelize.sync({ force, alter: false })
   }
 

server/models/Book.js

@@ -148,7 +148,7 @@ class Book extends Model {
         },
         title: DataTypes.STRING,
         titleIgnorePrefix: DataTypes.STRING,
-        subtitle: DataTypes.STRING,
+        subtitle: DataTypes.TEXT,
         publishedYear: DataTypes.STRING,
         publishedDate: DataTypes.STRING,
         publisher: DataTypes.STRING,

server/models/PlaybackSession.js

@@ -190,7 +190,7 @@ class PlaybackSession extends Model {
         currentTime: DataTypes.FLOAT,
         serverVersion: DataTypes.STRING,
         coverPath: DataTypes.STRING,
-        timeListening: DataTypes.INTEGER,
+        timeListening: DataTypes.FLOAT,
         mediaMetadata: DataTypes.JSON,
         date: DataTypes.STRING,
         dayOfWeek: DataTypes.STRING,

server/scripts/migrateSqliteToPostgres.js

@@ -9,11 +9,6 @@ const PG_SCHEMA = process.env.PG_SCHEMA || 'public'
 const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500)
 const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true'
 
-if (!DATABASE_URL) {
-  console.error('[migrate] DATABASE_URL is required')
-  process.exit(1)
-}
-
 const preferredOrder = [
   'migrationsMeta',
   'SequelizeMeta',
@@ -75,6 +70,120 @@ function sqliteGet(db, sql, params = []) {
   })
 }
 
+async function findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) {
+  const issues = []
+
+  for (const { sqliteTable, postgresTable } of tablesToMigrate) {
+    const pgColumns = pgColumnsByTable.get(postgresTable)
+    if (!pgColumns) continue
+
+    for (const column of pgColumns.values()) {
+      if (column.data_type !== 'character varying' || !column.character_maximum_length) continue
+
+      const sqliteColumn = column.column_name
+      const maxLength = Number(column.character_maximum_length)
+      const quotedTable = quoteIdent(sqliteTable)
+      const quotedColumn = quoteIdent(sqliteColumn)
+
+      try {
+        const maxLengthRow = await sqliteGet(
+          sqliteDb,
+          `SELECT MAX(LENGTH(${quotedColumn})) AS maxLength FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`
+        )
+        const actualMaxLength = Number(maxLengthRow?.maxLength || 0)
+        if (actualMaxLength <= maxLength) continue
+
+        const overCountRow = await sqliteGet(
+          sqliteDb,
+          `SELECT COUNT(*) AS count FROM ${quotedTable} WHERE LENGTH(${quotedColumn}) > ?`,
+          [maxLength]
+        )
+
+        issues.push({
+          sqliteTable,
+          sqliteColumn,
+          postgresTable,
+          postgresColumn: column.column_name,
+          maxLength,
+          actualMaxLength,
+          overCount: Number(overCountRow?.count || 0)
+        })
+      } catch (error) {
+        // Ignore columns missing in sqlite source table
+      }
+    }
+  }
+
+  return issues
+}
+
+function isIntegerCompatible(value) {
+  if (value === null || value === undefined) return true
+
+  if (typeof value === 'number') {
+    return Number.isFinite(value) && Number.isInteger(value)
+  }
+
+  if (typeof value === 'string') {
+    const trimmed = value.trim()
+    if (!trimmed) return false
+    if (!/^-?\d+$/.test(trimmed)) return false
+    const parsed = Number(trimmed)
+    return Number.isFinite(parsed)
+  }
+
+  return false
+}
+
+async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) {
+  const issues = []
+
+  for (const { sqliteTable, postgresTable } of tablesToMigrate) {
+    const pgColumns = pgColumnsByTable.get(postgresTable)
+    if (!pgColumns) continue
+
+    for (const column of pgColumns.values()) {
+      const dataType = column.data_type
+      if (dataType !== 'smallint' && dataType !== 'integer' && dataType !== 'bigint') continue
+
+      const sqliteColumn = column.column_name
+      const quotedTable = quoteIdent(sqliteTable)
+      const quotedColumn = quoteIdent(sqliteColumn)
+
+      let rows
+      try {
+        rows = await sqliteAll(sqliteDb, `SELECT ${quotedColumn} AS value FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`)
+      } catch (error) {
+        // Ignore columns missing in sqlite source table
+        continue
+      }
+
+      let badCount = 0
+      let sampleValue = null
+      for (const row of rows) {
+        if (!isIntegerCompatible(row.value)) {
+          badCount += 1
+          if (sampleValue === null) sampleValue = row.value
+        }
+      }
+
+      if (badCount > 0) {
+        issues.push({
+          sqliteTable,
+          sqliteColumn,
+          postgresTable,
+          postgresColumn: column.column_name,
+          postgresType: dataType,
+          badCount,
+          sampleValue
+        })
+      }
+    }
+  }
+
+  return issues
+}
+
 function normalizeBoolean(value) {
   if (value === null || value === undefined) return null
   if (typeof value === 'boolean') return value
@@ -88,7 +197,7 @@ function normalizeBoolean(value) {
 
 function normalizeJson(value) {
   if (value === null || value === undefined || value === '') return null
-  if (typeof value === 'object') return value
+  if (typeof value === 'object') return JSON.stringify(value)
 
   const textValue = String(value)
 
@@ -96,14 +205,14 @@ function normalizeJson(value) {
     const parsed = JSON.parse(textValue)
     if (typeof parsed === 'string') {
       try {
-        return JSON.parse(parsed)
+        return JSON.stringify(JSON.parse(parsed))
       } catch (error) {
-        return parsed
+        return JSON.stringify(parsed)
       }
     }
-    return parsed
+    return JSON.stringify(parsed)
   } catch (error) {
-    // Keep non-JSON payloads as JSON string values so inserts remain valid.
+    // Keep non-JSON payloads as JSON string values so inserts remain valid JSON.
     return JSON.stringify(textValue)
   }
 }
@@ -121,10 +230,45 @@ function convertValue(value, pgColumn) {
     return normalizeJson(value)
   }
 
+  if ((dataType === 'smallint' || dataType === 'integer' || dataType === 'bigint') && value !== null && value !== undefined) {
+    if (typeof value === 'number') return value
+    if (typeof value === 'string' && /^-?\d+$/.test(value.trim())) {
+      return Number(value)
+    }
+  }
+
   return value
 }
 
+function getOverlongColumns(row, insertColumns) {
+  const overlong = []
+
+  for (const column of insertColumns) {
+    const maxLength = column.metadata.character_maximum_length
+    if (!maxLength) continue
+
+    const value = row[column.sqliteColumn]
+    if (value === null || value === undefined) continue
+
+    const length = String(value).length
+    if (length > maxLength) {
+      overlong.push({
+        sqliteColumn: column.sqliteColumn,
+        postgresColumn: column.postgresColumn,
+        maxLength,
+        actualLength: length
+      })
+    }
+  }
+
+  return overlong
+}
+
 async function main() {
+  if (!DATABASE_URL) {
+    throw new Error('DATABASE_URL is required')
+  }
+
   console.log(`[migrate] sqlite source: ${SQLITE_PATH}`)
   console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`)
   console.log(`[migrate] dry run: ${DRY_RUN}`)
@@ -176,13 +320,38 @@ async function main() {
     const pgColumnsByTable = new Map()
     for (const { postgresTable } of tablesToMigrate) {
       const columnsResult = await pg.query(
-        `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`,
+        `SELECT column_name, data_type, udt_name, character_maximum_length FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`,
         [PG_SCHEMA, postgresTable]
       )
       const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column]))
       pgColumnsByTable.set(postgresTable, columnMap)
     }
 
+    const overlongVarcharIssues = await findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable)
+    const integerTypeIssues = await findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable)
+
+    if (overlongVarcharIssues.length) {
+      console.error('[migrate] overlong source values detected for varchar columns:')
+      overlongVarcharIssues.forEach((issue) => {
+        console.error(
+          `[migrate]   ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` +
+            `(max=${issue.maxLength}, actualMax=${issue.actualMaxLength}, overRows=${issue.overCount})`
+        )
+      })
+      throw new Error('Migration aborted to prevent truncation/data loss. Widen target column types first.')
+    }
+
+    if (integerTypeIssues.length) {
+      console.error('[migrate] non-integer source values detected for integer columns:')
+      integerTypeIssues.forEach((issue) => {
+        console.error(
+          `[migrate]   ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` +
+            `(type=${issue.postgresType}, badRows=${issue.badCount}, sample=${JSON.stringify(issue.sampleValue)})`
+        )
+      })
+      throw new Error('Migration aborted to prevent numeric precision loss. Widen target numeric column types first.')
+    }
+
     if (!DRY_RUN) {
       await pg.query('BEGIN')
       await pg.query('SET session_replication_role = replica')
@@ -231,7 +400,40 @@ async function main() {
           }
 
           const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}`
+
+          try {
+            await pg.query('SAVEPOINT migrate_batch')
             await pg.query(insertSql, params)
+            await pg.query('RELEASE SAVEPOINT migrate_batch')
+          } catch (error) {
+            await pg.query('ROLLBACK TO SAVEPOINT migrate_batch')
+            console.error(`[migrate] batch insert failed for ${sqliteTable} (offset=${offset}, size=${batchRows.length}): ${error.message}`)
+
+            for (let rowIndex = 0; rowIndex < batchRows.length; rowIndex++) {
+              const row = batchRows[rowIndex]
+              const singleRowParams = insertColumns.map((column) => convertValue(row[column.sqliteColumn], column.metadata))
+              const singleRowInsertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES (${singleRowParams.map((_, index) => `$${index + 1}`).join(', ')})`
+
+              try {
+                await pg.query('SAVEPOINT migrate_row')
+                await pg.query(singleRowInsertSql, singleRowParams)
+                await pg.query('RELEASE SAVEPOINT migrate_row')
+              } catch (rowError) {
+                await pg.query('ROLLBACK TO SAVEPOINT migrate_row')
+                const overlongColumns = getOverlongColumns(row, insertColumns)
+                if (overlongColumns.length) {
+                  overlongColumns.forEach((column) => {
+                    console.error(
+                      `[migrate] overlong value in ${sqliteTable}.${column.sqliteColumn} -> ${postgresTable}.${column.postgresColumn} ` +
+                        `(length=${column.actualLength}, max=${column.maxLength})`
+                    )
+                  })
+                }
+
+                throw rowError
+              }
+            }
+          }
         }
       }
 
@@ -271,7 +473,18 @@ async function main() {
   }
 }
 
+if (require.main === module) {
   main().catch((error) => {
     console.error('[migrate] failed:', error)
     process.exit(1)
   })
+}
+
+module.exports = {
+  normalizeJson,
+  isIntegerCompatible,
+  convertValue,
+  findOverlongVarcharValues,
+  findIntegerTypeIssues,
+  quoteIdent
+}

test/server/scripts/migrateSqliteToPostgres.test.js

@@ -0,0 +1,131 @@
+const { expect } = require('chai')
+const sqlite3 = require('sqlite3')
+
+const {
+  normalizeJson,
+  isIntegerCompatible,
+  findOverlongVarcharValues,
+  findIntegerTypeIssues
+} = require('../../../server/scripts/migrateSqliteToPostgres')
+
+function openMemoryDb() {
+  return new Promise((resolve, reject) => {
+    const db = new sqlite3.Database(':memory:', (error) => {
+      if (error) return reject(error)
+      resolve(db)
+    })
+  })
+}
+
+function run(db, sql, params = []) {
+  return new Promise((resolve, reject) => {
+    db.run(sql, params, (error) => {
+      if (error) return reject(error)
+      resolve()
+    })
+  })
+}
+
+function close(db) {
+  return new Promise((resolve, reject) => {
+    db.close((error) => {
+      if (error) return reject(error)
+      resolve()
+    })
+  })
+}
+
+describe('migrateSqliteToPostgres script helpers', () => {
+  it('should keep malformed JSON payloads insertable by returning valid JSON text', () => {
+    const malformed = '"{"x",1}"'
+    const normalized = normalizeJson(malformed)
+
+    expect(() => JSON.parse(normalized)).to.not.throw()
+    expect(JSON.parse(normalized)).to.equal('"{"x",1}"')
+  })
+
+  it('should detect overlong varchar values before migration', async () => {
+    const db = await openMemoryDb()
+
+    try {
+      await run(db, 'CREATE TABLE books (subtitle TEXT)')
+      await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['x'.repeat(300)])
+      await run(db, 'INSERT INTO books (subtitle) VALUES (?)', ['ok'])
+
+      const tablesToMigrate = [{ sqliteTable: 'books', postgresTable: 'books' }]
+      const pgColumnsByTable = new Map([
+        [
+          'books',
+          new Map([
+            [
+              'subtitle',
+              {
+                column_name: 'subtitle',
+                data_type: 'character varying',
+                character_maximum_length: 255
+              }
+            ]
+          ])
+        ]
+      ])
+
+      const issues = await findOverlongVarcharValues(db, tablesToMigrate, pgColumnsByTable)
+
+      expect(issues).to.have.length(1)
+      expect(issues[0]).to.include({
+        sqliteTable: 'books',
+        sqliteColumn: 'subtitle',
+        postgresTable: 'books',
+        postgresColumn: 'subtitle',
+        maxLength: 255,
+        actualMaxLength: 300,
+        overCount: 1
+      })
+    } finally {
+      await close(db)
+    }
+  })
+
+  it('should detect non-integer values for integer target columns', async () => {
+    const db = await openMemoryDb()
+
+    try {
+      await run(db, 'CREATE TABLE playbackSessions (timeListening REAL)')
+      await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [25.802536999999997])
+      await run(db, 'INSERT INTO playbackSessions (timeListening) VALUES (?)', [42])
+
+      const tablesToMigrate = [{ sqliteTable: 'playbackSessions', postgresTable: 'playbackSessions' }]
+      const pgColumnsByTable = new Map([
+        [
+          'playbackSessions',
+          new Map([
+            [
+              'timelistening',
+              {
+                column_name: 'timeListening',
+                data_type: 'integer'
+              }
+            ]
+          ])
+        ]
+      ])
+
+      const issues = await findIntegerTypeIssues(db, tablesToMigrate, pgColumnsByTable)
+
+      expect(issues).to.have.length(1)
+      expect(issues[0].sqliteTable).to.equal('playbackSessions')
+      expect(issues[0].sqliteColumn).to.equal('timeListening')
+      expect(issues[0].postgresType).to.equal('integer')
+      expect(issues[0].badCount).to.equal(1)
+    } finally {
+      await close(db)
+    }
+  })
+
+  it('should only accept integer-compatible values for integer columns', () => {
+    expect(isIntegerCompatible(10)).to.equal(true)
+    expect(isIntegerCompatible('10')).to.equal(true)
+    expect(isIntegerCompatible(10.5)).to.equal(false)
+    expect(isIntegerCompatible('10.5')).to.equal(false)
+  })
+})