groove/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2026-07-07 09:51:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

make sqlite to postgres migrator case-insensitive

Browse source
This commit is contained in:
Kevin Gatera 2026-03-02 17:12:12 -05:00
parent c7d14f7d30
commit c0a4ce1396
No known key found for this signature in database
GPG key ID: F0D9F5932458CFB9
1 changed files with 46 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

70
server/scripts/migrateSqliteToPostgres.js
View file

@ -132,13 +132,24 @@ async function main() {
`SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`, `SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`,
[PG_SCHEMA] [PG_SCHEMA]
) )
const pgTables = new Set(pgTablesRows.rows.map((row) => row.table_name))
const tablesToMigrate = sqliteTables.filter((table) => pgTables.has(table)) const pgTablesByLowerName = new Map(pgTablesRows.rows.map((row) => [row.table_name.toLowerCase(), row.table_name]))
const tablesToMigrate = sqliteTables
.map((sqliteTable) => {
const postgresTable = pgTablesByLowerName.get(sqliteTable.toLowerCase())
if (!postgresTable) return null
return {
sqliteTable,
postgresTable
}
})
.filter(Boolean)
tablesToMigrate.sort((a, b) => { tablesToMigrate.sort((a, b) => {
const ai = preferredOrder.indexOf(a) const ai = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === a.sqliteTable.toLowerCase())
const bi = preferredOrder.indexOf(b) const bi = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === b.sqliteTable.toLowerCase())
if (ai === -1 && bi === -1) return a.localeCompare(b) if (ai === -1 && bi === -1) return a.sqliteTable.localeCompare(b.sqliteTable)
if (ai === -1) return 1 if (ai === -1) return 1
if (bi === -1) return -1 if (bi === -1) return -1
return ai - bi return ai - bi
@ -148,34 +159,46 @@ async function main() {
throw new Error('No overlapping tables found between SQLite and PostgreSQL') throw new Error('No overlapping tables found between SQLite and PostgreSQL')
} }
console.log(`[migrate] tables to migrate: ${tablesToMigrate.join(', ')}`) console.log(`[migrate] tables to migrate: ${tablesToMigrate.map((table) => table.sqliteTable).join(', ')}`)
const pgColumnsByTable = new Map() const pgColumnsByTable = new Map()
for (const table of tablesToMigrate) { for (const { postgresTable } of tablesToMigrate) {
const columnsResult = await pg.query( const columnsResult = await pg.query(
`SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`, `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`,
[PG_SCHEMA, table] [PG_SCHEMA, postgresTable]
) )
const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name, column])) const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column]))
pgColumnsByTable.set(table, columnMap) pgColumnsByTable.set(postgresTable, columnMap)
} }
if (!DRY_RUN) { if (!DRY_RUN) {
await pg.query('BEGIN') await pg.query('BEGIN')
await pg.query('SET session_replication_role = replica') await pg.query('SET session_replication_role = replica')
const truncateList = tablesToMigrate.map((table) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`).join(', ') const truncateList = tablesToMigrate.map(({ postgresTable }) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`).join(', ')
await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`) await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`)
console.log('[migrate] truncated target tables') console.log('[migrate] truncated target tables')
} }
for (const table of tablesToMigrate) { for (const { sqliteTable, postgresTable } of tablesToMigrate) {
const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(table)}`) const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(sqliteTable)}`)
const pgColumns = pgColumnsByTable.get(table) const pgColumns = pgColumnsByTable.get(postgresTable)
const insertColumns = rows.length ? Object.keys(rows[0]).filter((column) => pgColumns.has(column)) : [] const insertColumns = rows.length
? Object.keys(rows[0])
.map((sqliteColumn) => {
const pgColumn = pgColumns.get(sqliteColumn.toLowerCase())
if (!pgColumn) return null
return {
sqliteColumn,
postgresColumn: pgColumn.column_name,
metadata: pgColumn
}
})
.filter(Boolean)
: []
if (!rows.length || !insertColumns.length) { if (!rows.length || !insertColumns.length) {
console.log(`[migrate] ${table}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`) console.log(`[migrate] ${sqliteTable}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`)
continue continue
} }
@ -189,19 +212,18 @@ async function main() {
for (const row of batchRows) { for (const row of batchRows) {
const placeholders = [] const placeholders = []
for (const column of insertColumns) { for (const column of insertColumns) {
const pgColumn = pgColumns.get(column) params.push(convertValue(row[column.sqliteColumn], column.metadata))
params.push(convertValue(row[column], pgColumn))
placeholders.push(`$${paramIndex++}`) placeholders.push(`$${paramIndex++}`)
} }
valuesSql.push(`(${placeholders.join(', ')})`) valuesSql.push(`(${placeholders.join(', ')})`)
} }
const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)} (${insertColumns.map(quoteIdent).join(', ')}) VALUES ${valuesSql.join(', ')}` const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}`
await pg.query(insertSql, params) await pg.query(insertSql, params)
} }
} }
console.log(`[migrate] ${table}: ${rows.length} rows`) console.log(`[migrate] ${sqliteTable}: ${rows.length} rows`)
} }
if (!DRY_RUN) { if (!DRY_RUN) {
@ -211,11 +233,11 @@ async function main() {
} }
const parity = [] const parity = []
for (const table of tablesToMigrate) { for (const { sqliteTable, postgresTable } of tablesToMigrate) {
const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(table)}`) const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(sqliteTable)}`)
const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(table)}`) const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`)
parity.push({ parity.push({
table, table: sqliteTable,
sqliteCount: Number(sqliteCountRow.count || 0), sqliteCount: Number(sqliteCountRow.count || 0),
postgresCount: Number(pgCountResult.rows[0].count || 0) postgresCount: Number(pgCountResult.rows[0].count || 0)
}) })