add postgres dialect support for homelab migration

server/scripts/migrateSqliteToPostgres.js

@@ -0,0 +1,243 @@
+#!/usr/bin/env node
+
+const sqlite3 = require('sqlite3')
+const { Client } = require('pg')
+
+const SQLITE_PATH = process.env.SQLITE_PATH || '/config/absdatabase.sqlite'
+const DATABASE_URL = process.env.DATABASE_URL
+const PG_SCHEMA = process.env.PG_SCHEMA || 'public'
+const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500)
+const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true'
+
+if (!DATABASE_URL) {
+  console.error('[migrate] DATABASE_URL is required')
+  process.exit(1)
+}
+
+const preferredOrder = [
+  'migrationsMeta',
+  'SequelizeMeta',
+  'settings',
+  'users',
+  'apiKeys',
+  'sessions',
+  'libraries',
+  'libraryFolders',
+  'authors',
+  'series',
+  'books',
+  'podcasts',
+  'podcastEpisodes',
+  'libraryItems',
+  'bookAuthors',
+  'bookSeries',
+  'collections',
+  'collectionBooks',
+  'playlists',
+  'playlistMediaItems',
+  'mediaProgresses',
+  'devices',
+  'playbackSessions',
+  'feeds',
+  'feedEpisodes',
+  'mediaItemShares',
+  'customMetadataProviders'
+]
+
+function quoteIdent(identifier) {
+  return `"${String(identifier).replace(/"/g, '""')}"`
+}
+
+function openSqlite(filePath) {
+  return new Promise((resolve, reject) => {
+    const db = new sqlite3.Database(filePath, sqlite3.OPEN_READONLY, (err) => {
+      if (err) return reject(err)
+      resolve(db)
+    })
+  })
+}
+
+function sqliteAll(db, sql, params = []) {
+  return new Promise((resolve, reject) => {
+    db.all(sql, params, (err, rows) => {
+      if (err) return reject(err)
+      resolve(rows)
+    })
+  })
+}
+
+function sqliteGet(db, sql, params = []) {
+  return new Promise((resolve, reject) => {
+    db.get(sql, params, (err, row) => {
+      if (err) return reject(err)
+      resolve(row)
+    })
+  })
+}
+
+function normalizeBoolean(value) {
+  if (value === null || value === undefined) return null
+  if (typeof value === 'boolean') return value
+  if (typeof value === 'number') return value !== 0
+  if (typeof value === 'string') {
+    const v = value.trim().toLowerCase()
+    return v === 'true' || v === '1' || v === 't'
+  }
+  return !!value
+}
+
+function normalizeJson(value) {
+  if (value === null || value === undefined || value === '') return null
+  if (typeof value === 'object') return value
+  try {
+    return JSON.parse(value)
+  } catch (error) {
+    return value
+  }
+}
+
+function convertValue(value, pgColumn) {
+  if (!pgColumn) return value
+  const dataType = pgColumn.data_type
+  const udtName = pgColumn.udt_name
+
+  if (dataType === 'boolean') {
+    return normalizeBoolean(value)
+  }
+
+  if (dataType === 'json' || dataType === 'jsonb' || udtName === 'json' || udtName === 'jsonb') {
+    return normalizeJson(value)
+  }
+
+  return value
+}
+
+async function main() {
+  console.log(`[migrate] sqlite source: ${SQLITE_PATH}`)
+  console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`)
+  console.log(`[migrate] dry run: ${DRY_RUN}`)
+
+  const sqliteDb = await openSqlite(SQLITE_PATH)
+  const pg = new Client({ connectionString: DATABASE_URL })
+  await pg.connect()
+
+  try {
+    const sqliteTablesRows = await sqliteAll(
+      sqliteDb,
+      "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
+    )
+    const sqliteTables = sqliteTablesRows.map((row) => row.name)
+
+    const pgTablesRows = await pg.query(
+      `SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`,
+      [PG_SCHEMA]
+    )
+    const pgTables = new Set(pgTablesRows.rows.map((row) => row.table_name))
+
+    const tablesToMigrate = sqliteTables.filter((table) => pgTables.has(table))
+    tablesToMigrate.sort((a, b) => {
+      const ai = preferredOrder.indexOf(a)
+      const bi = preferredOrder.indexOf(b)
+      if (ai === -1 && bi === -1) return a.localeCompare(b)
+      if (ai === -1) return 1
+      if (bi === -1) return -1
+      return ai - bi
+    })
+
+    if (!tablesToMigrate.length) {
+      throw new Error('No overlapping tables found between SQLite and PostgreSQL')
+    }
+
+    console.log(`[migrate] tables to migrate: ${tablesToMigrate.join(', ')}`)
+
+    const pgColumnsByTable = new Map()
+    for (const table of tablesToMigrate) {
+      const columnsResult = await pg.query(
+        `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`,
+        [PG_SCHEMA, table]
+      )
+      const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name, column]))
+      pgColumnsByTable.set(table, columnMap)
+    }
+
+    if (!DRY_RUN) {
+      await pg.query('BEGIN')
+      await pg.query('SET session_replication_role = replica')
+
+      const truncateList = tablesToMigrate.map((table) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`).join(', ')
+      await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`)
+      console.log('[migrate] truncated target tables')
+    }
+
+    for (const table of tablesToMigrate) {
+      const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(table)}`)
+      const pgColumns = pgColumnsByTable.get(table)
+      const insertColumns = rows.length ? Object.keys(rows[0]).filter((column) => pgColumns.has(column)) : []
+
+      if (!rows.length || !insertColumns.length) {
+        console.log(`[migrate] ${table}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`)
+        continue
+      }
+
+      if (!DRY_RUN) {
+        for (let offset = 0; offset < rows.length; offset += BATCH_SIZE) {
+          const batchRows = rows.slice(offset, offset + BATCH_SIZE)
+          const valuesSql = []
+          const params = []
+          let paramIndex = 1
+
+          for (const row of batchRows) {
+            const placeholders = []
+            for (const column of insertColumns) {
+              const pgColumn = pgColumns.get(column)
+              params.push(convertValue(row[column], pgColumn))
+              placeholders.push(`$${paramIndex++}`)
+            }
+            valuesSql.push(`(${placeholders.join(', ')})`)
+          }
+
+          const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)} (${insertColumns.map(quoteIdent).join(', ')}) VALUES ${valuesSql.join(', ')}`
+          await pg.query(insertSql, params)
+        }
+      }
+
+      console.log(`[migrate] ${table}: ${rows.length} rows`)
+    }
+
+    if (!DRY_RUN) {
+      await pg.query('SET session_replication_role = DEFAULT')
+      await pg.query('COMMIT')
+      console.log('[migrate] migration transaction committed')
+    }
+
+    const parity = []
+    for (const table of tablesToMigrate) {
+      const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(table)}`)
+      const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`)
+      parity.push({
+        table,
+        sqliteCount: Number(sqliteCountRow.count || 0),
+        postgresCount: Number(pgCountResult.rows[0].count || 0)
+      })
+    }
+
+    const mismatches = parity.filter((row) => row.sqliteCount !== row.postgresCount)
+    if (mismatches.length) {
+      console.error('[migrate] row-count mismatches detected:')
+      mismatches.forEach((row) => {
+        console.error(`[migrate]   ${row.table}: sqlite=${row.sqliteCount} postgres=${row.postgresCount}`)
+      })
+      process.exitCode = 2
+    } else {
+      console.log('[migrate] parity check passed for all migrated tables')
+    }
+  } finally {
+    sqliteDb.close()
+    await pg.end()
+  }
+}
+
+main().catch((error) => {
+  console.error('[migrate] failed:', error)
+  process.exit(1)
+})