mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2026-07-07 09:51:37 +00:00
490 lines
16 KiB
JavaScript
490 lines
16 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
const sqlite3 = require('sqlite3')
|
|
const { Client } = require('pg')
|
|
|
|
const SQLITE_PATH = process.env.SQLITE_PATH || '/config/absdatabase.sqlite'
|
|
const DATABASE_URL = process.env.DATABASE_URL
|
|
const PG_SCHEMA = process.env.PG_SCHEMA || 'public'
|
|
const BATCH_SIZE = Number(process.env.MIGRATION_BATCH_SIZE || 500)
|
|
const DRY_RUN = String(process.env.DRY_RUN || 'false').toLowerCase() === 'true'
|
|
|
|
const preferredOrder = [
|
|
'migrationsMeta',
|
|
'SequelizeMeta',
|
|
'settings',
|
|
'users',
|
|
'apiKeys',
|
|
'sessions',
|
|
'libraries',
|
|
'libraryFolders',
|
|
'authors',
|
|
'series',
|
|
'books',
|
|
'podcasts',
|
|
'podcastEpisodes',
|
|
'libraryItems',
|
|
'bookAuthors',
|
|
'bookSeries',
|
|
'collections',
|
|
'collectionBooks',
|
|
'playlists',
|
|
'playlistMediaItems',
|
|
'mediaProgresses',
|
|
'devices',
|
|
'playbackSessions',
|
|
'feeds',
|
|
'feedEpisodes',
|
|
'mediaItemShares',
|
|
'customMetadataProviders'
|
|
]
|
|
|
|
function quoteIdent(identifier) {
|
|
return `"${String(identifier).replace(/"/g, '""')}"`
|
|
}
|
|
|
|
function openSqlite(filePath) {
|
|
return new Promise((resolve, reject) => {
|
|
const db = new sqlite3.Database(filePath, sqlite3.OPEN_READONLY, (err) => {
|
|
if (err) return reject(err)
|
|
resolve(db)
|
|
})
|
|
})
|
|
}
|
|
|
|
function sqliteAll(db, sql, params = []) {
|
|
return new Promise((resolve, reject) => {
|
|
db.all(sql, params, (err, rows) => {
|
|
if (err) return reject(err)
|
|
resolve(rows)
|
|
})
|
|
})
|
|
}
|
|
|
|
function sqliteGet(db, sql, params = []) {
|
|
return new Promise((resolve, reject) => {
|
|
db.get(sql, params, (err, row) => {
|
|
if (err) return reject(err)
|
|
resolve(row)
|
|
})
|
|
})
|
|
}
|
|
|
|
async function findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable) {
|
|
const issues = []
|
|
|
|
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
|
|
const pgColumns = pgColumnsByTable.get(postgresTable)
|
|
if (!pgColumns) continue
|
|
|
|
for (const column of pgColumns.values()) {
|
|
if (column.data_type !== 'character varying' || !column.character_maximum_length) continue
|
|
|
|
const sqliteColumn = column.column_name
|
|
const maxLength = Number(column.character_maximum_length)
|
|
const quotedTable = quoteIdent(sqliteTable)
|
|
const quotedColumn = quoteIdent(sqliteColumn)
|
|
|
|
try {
|
|
const maxLengthRow = await sqliteGet(
|
|
sqliteDb,
|
|
`SELECT MAX(LENGTH(${quotedColumn})) AS maxLength FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`
|
|
)
|
|
const actualMaxLength = Number(maxLengthRow?.maxLength || 0)
|
|
if (actualMaxLength <= maxLength) continue
|
|
|
|
const overCountRow = await sqliteGet(
|
|
sqliteDb,
|
|
`SELECT COUNT(*) AS count FROM ${quotedTable} WHERE LENGTH(${quotedColumn}) > ?`,
|
|
[maxLength]
|
|
)
|
|
|
|
issues.push({
|
|
sqliteTable,
|
|
sqliteColumn,
|
|
postgresTable,
|
|
postgresColumn: column.column_name,
|
|
maxLength,
|
|
actualMaxLength,
|
|
overCount: Number(overCountRow?.count || 0)
|
|
})
|
|
} catch (error) {
|
|
// Ignore columns missing in sqlite source table
|
|
}
|
|
}
|
|
}
|
|
|
|
return issues
|
|
}
|
|
|
|
function isIntegerCompatible(value) {
|
|
if (value === null || value === undefined) return true
|
|
|
|
if (typeof value === 'number') {
|
|
return Number.isFinite(value) && Number.isInteger(value)
|
|
}
|
|
|
|
if (typeof value === 'string') {
|
|
const trimmed = value.trim()
|
|
if (!trimmed) return false
|
|
if (!/^-?\d+$/.test(trimmed)) return false
|
|
const parsed = Number(trimmed)
|
|
return Number.isFinite(parsed)
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
async function findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable) {
|
|
const issues = []
|
|
|
|
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
|
|
const pgColumns = pgColumnsByTable.get(postgresTable)
|
|
if (!pgColumns) continue
|
|
|
|
for (const column of pgColumns.values()) {
|
|
const dataType = column.data_type
|
|
if (dataType !== 'smallint' && dataType !== 'integer' && dataType !== 'bigint') continue
|
|
|
|
const sqliteColumn = column.column_name
|
|
const quotedTable = quoteIdent(sqliteTable)
|
|
const quotedColumn = quoteIdent(sqliteColumn)
|
|
|
|
let rows
|
|
try {
|
|
rows = await sqliteAll(sqliteDb, `SELECT ${quotedColumn} AS value FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`)
|
|
} catch (error) {
|
|
// Ignore columns missing in sqlite source table
|
|
continue
|
|
}
|
|
|
|
let badCount = 0
|
|
let sampleValue = null
|
|
for (const row of rows) {
|
|
if (!isIntegerCompatible(row.value)) {
|
|
badCount += 1
|
|
if (sampleValue === null) sampleValue = row.value
|
|
}
|
|
}
|
|
|
|
if (badCount > 0) {
|
|
issues.push({
|
|
sqliteTable,
|
|
sqliteColumn,
|
|
postgresTable,
|
|
postgresColumn: column.column_name,
|
|
postgresType: dataType,
|
|
badCount,
|
|
sampleValue
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
return issues
|
|
}
|
|
|
|
function normalizeBoolean(value) {
|
|
if (value === null || value === undefined) return null
|
|
if (typeof value === 'boolean') return value
|
|
if (typeof value === 'number') return value !== 0
|
|
if (typeof value === 'string') {
|
|
const v = value.trim().toLowerCase()
|
|
return v === 'true' || v === '1' || v === 't'
|
|
}
|
|
return !!value
|
|
}
|
|
|
|
function normalizeJson(value) {
|
|
if (value === null || value === undefined || value === '') return null
|
|
if (typeof value === 'object') return JSON.stringify(value)
|
|
|
|
const textValue = String(value)
|
|
|
|
try {
|
|
const parsed = JSON.parse(textValue)
|
|
if (typeof parsed === 'string') {
|
|
try {
|
|
return JSON.stringify(JSON.parse(parsed))
|
|
} catch (error) {
|
|
return JSON.stringify(parsed)
|
|
}
|
|
}
|
|
return JSON.stringify(parsed)
|
|
} catch (error) {
|
|
// Keep non-JSON payloads as JSON string values so inserts remain valid JSON.
|
|
return JSON.stringify(textValue)
|
|
}
|
|
}
|
|
|
|
function convertValue(value, pgColumn) {
|
|
if (!pgColumn) return value
|
|
const dataType = pgColumn.data_type
|
|
const udtName = pgColumn.udt_name
|
|
|
|
if (dataType === 'boolean') {
|
|
return normalizeBoolean(value)
|
|
}
|
|
|
|
if (dataType === 'json' || dataType === 'jsonb' || udtName === 'json' || udtName === 'jsonb') {
|
|
return normalizeJson(value)
|
|
}
|
|
|
|
if ((dataType === 'smallint' || dataType === 'integer' || dataType === 'bigint') && value !== null && value !== undefined) {
|
|
if (typeof value === 'number') return value
|
|
if (typeof value === 'string' && /^-?\d+$/.test(value.trim())) {
|
|
return Number(value)
|
|
}
|
|
}
|
|
|
|
return value
|
|
}
|
|
|
|
function getOverlongColumns(row, insertColumns) {
|
|
const overlong = []
|
|
|
|
for (const column of insertColumns) {
|
|
const maxLength = column.metadata.character_maximum_length
|
|
if (!maxLength) continue
|
|
|
|
const value = row[column.sqliteColumn]
|
|
if (value === null || value === undefined) continue
|
|
|
|
const length = String(value).length
|
|
if (length > maxLength) {
|
|
overlong.push({
|
|
sqliteColumn: column.sqliteColumn,
|
|
postgresColumn: column.postgresColumn,
|
|
maxLength,
|
|
actualLength: length
|
|
})
|
|
}
|
|
}
|
|
|
|
return overlong
|
|
}
|
|
|
|
async function main() {
|
|
if (!DATABASE_URL) {
|
|
throw new Error('DATABASE_URL is required')
|
|
}
|
|
|
|
console.log(`[migrate] sqlite source: ${SQLITE_PATH}`)
|
|
console.log(`[migrate] postgres target schema: ${PG_SCHEMA}`)
|
|
console.log(`[migrate] dry run: ${DRY_RUN}`)
|
|
|
|
const sqliteDb = await openSqlite(SQLITE_PATH)
|
|
const pg = new Client({ connectionString: DATABASE_URL })
|
|
await pg.connect()
|
|
|
|
try {
|
|
const sqliteTablesRows = await sqliteAll(
|
|
sqliteDb,
|
|
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
|
)
|
|
const sqliteTables = sqliteTablesRows.map((row) => row.name)
|
|
|
|
const pgTablesRows = await pg.query(
|
|
`SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type='BASE TABLE' ORDER BY table_name`,
|
|
[PG_SCHEMA]
|
|
)
|
|
|
|
const pgTablesByLowerName = new Map(pgTablesRows.rows.map((row) => [row.table_name.toLowerCase(), row.table_name]))
|
|
|
|
const tablesToMigrate = sqliteTables
|
|
.map((sqliteTable) => {
|
|
const postgresTable = pgTablesByLowerName.get(sqliteTable.toLowerCase())
|
|
if (!postgresTable) return null
|
|
return {
|
|
sqliteTable,
|
|
postgresTable
|
|
}
|
|
})
|
|
.filter(Boolean)
|
|
|
|
tablesToMigrate.sort((a, b) => {
|
|
const ai = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === a.sqliteTable.toLowerCase())
|
|
const bi = preferredOrder.findIndex((tableName) => tableName.toLowerCase() === b.sqliteTable.toLowerCase())
|
|
if (ai === -1 && bi === -1) return a.sqliteTable.localeCompare(b.sqliteTable)
|
|
if (ai === -1) return 1
|
|
if (bi === -1) return -1
|
|
return ai - bi
|
|
})
|
|
|
|
if (!tablesToMigrate.length) {
|
|
throw new Error('No overlapping tables found between SQLite and PostgreSQL')
|
|
}
|
|
|
|
console.log(`[migrate] tables to migrate: ${tablesToMigrate.map((table) => table.sqliteTable).join(', ')}`)
|
|
|
|
const pgColumnsByTable = new Map()
|
|
for (const { postgresTable } of tablesToMigrate) {
|
|
const columnsResult = await pg.query(
|
|
`SELECT column_name, data_type, udt_name, character_maximum_length FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position`,
|
|
[PG_SCHEMA, postgresTable]
|
|
)
|
|
const columnMap = new Map(columnsResult.rows.map((column) => [column.column_name.toLowerCase(), column]))
|
|
pgColumnsByTable.set(postgresTable, columnMap)
|
|
}
|
|
|
|
const overlongVarcharIssues = await findOverlongVarcharValues(sqliteDb, tablesToMigrate, pgColumnsByTable)
|
|
const integerTypeIssues = await findIntegerTypeIssues(sqliteDb, tablesToMigrate, pgColumnsByTable)
|
|
|
|
if (overlongVarcharIssues.length) {
|
|
console.error('[migrate] overlong source values detected for varchar columns:')
|
|
overlongVarcharIssues.forEach((issue) => {
|
|
console.error(
|
|
`[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` +
|
|
`(max=${issue.maxLength}, actualMax=${issue.actualMaxLength}, overRows=${issue.overCount})`
|
|
)
|
|
})
|
|
throw new Error('Migration aborted to prevent truncation/data loss. Widen target column types first.')
|
|
}
|
|
|
|
if (integerTypeIssues.length) {
|
|
console.error('[migrate] non-integer source values detected for integer columns:')
|
|
integerTypeIssues.forEach((issue) => {
|
|
console.error(
|
|
`[migrate] ${issue.sqliteTable}.${issue.sqliteColumn} -> ${issue.postgresTable}.${issue.postgresColumn} ` +
|
|
`(type=${issue.postgresType}, badRows=${issue.badCount}, sample=${JSON.stringify(issue.sampleValue)})`
|
|
)
|
|
})
|
|
throw new Error('Migration aborted to prevent numeric precision loss. Widen target numeric column types first.')
|
|
}
|
|
|
|
if (!DRY_RUN) {
|
|
await pg.query('BEGIN')
|
|
await pg.query('SET session_replication_role = replica')
|
|
|
|
const truncateList = tablesToMigrate.map(({ postgresTable }) => `${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`).join(', ')
|
|
await pg.query(`TRUNCATE TABLE ${truncateList} RESTART IDENTITY CASCADE`)
|
|
console.log('[migrate] truncated target tables')
|
|
}
|
|
|
|
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
|
|
const rows = await sqliteAll(sqliteDb, `SELECT * FROM ${quoteIdent(sqliteTable)}`)
|
|
const pgColumns = pgColumnsByTable.get(postgresTable)
|
|
const insertColumns = rows.length
|
|
? Object.keys(rows[0])
|
|
.map((sqliteColumn) => {
|
|
const pgColumn = pgColumns.get(sqliteColumn.toLowerCase())
|
|
if (!pgColumn) return null
|
|
return {
|
|
sqliteColumn,
|
|
postgresColumn: pgColumn.column_name,
|
|
metadata: pgColumn
|
|
}
|
|
})
|
|
.filter(Boolean)
|
|
: []
|
|
|
|
if (!rows.length || !insertColumns.length) {
|
|
console.log(`[migrate] ${sqliteTable}: skipped (rows=${rows.length}, insertableColumns=${insertColumns.length})`)
|
|
continue
|
|
}
|
|
|
|
if (!DRY_RUN) {
|
|
for (let offset = 0; offset < rows.length; offset += BATCH_SIZE) {
|
|
const batchRows = rows.slice(offset, offset + BATCH_SIZE)
|
|
const valuesSql = []
|
|
const params = []
|
|
let paramIndex = 1
|
|
|
|
for (const row of batchRows) {
|
|
const placeholders = []
|
|
for (const column of insertColumns) {
|
|
params.push(convertValue(row[column.sqliteColumn], column.metadata))
|
|
placeholders.push(`$${paramIndex++}`)
|
|
}
|
|
valuesSql.push(`(${placeholders.join(', ')})`)
|
|
}
|
|
|
|
const insertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES ${valuesSql.join(', ')}`
|
|
|
|
try {
|
|
await pg.query('SAVEPOINT migrate_batch')
|
|
await pg.query(insertSql, params)
|
|
await pg.query('RELEASE SAVEPOINT migrate_batch')
|
|
} catch (error) {
|
|
await pg.query('ROLLBACK TO SAVEPOINT migrate_batch')
|
|
console.error(`[migrate] batch insert failed for ${sqliteTable} (offset=${offset}, size=${batchRows.length}): ${error.message}`)
|
|
|
|
for (let rowIndex = 0; rowIndex < batchRows.length; rowIndex++) {
|
|
const row = batchRows[rowIndex]
|
|
const singleRowParams = insertColumns.map((column) => convertValue(row[column.sqliteColumn], column.metadata))
|
|
const singleRowInsertSql = `INSERT INTO ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)} (${insertColumns.map((column) => quoteIdent(column.postgresColumn)).join(', ')}) VALUES (${singleRowParams.map((_, index) => `$${index + 1}`).join(', ')})`
|
|
|
|
try {
|
|
await pg.query('SAVEPOINT migrate_row')
|
|
await pg.query(singleRowInsertSql, singleRowParams)
|
|
await pg.query('RELEASE SAVEPOINT migrate_row')
|
|
} catch (rowError) {
|
|
await pg.query('ROLLBACK TO SAVEPOINT migrate_row')
|
|
const overlongColumns = getOverlongColumns(row, insertColumns)
|
|
if (overlongColumns.length) {
|
|
overlongColumns.forEach((column) => {
|
|
console.error(
|
|
`[migrate] overlong value in ${sqliteTable}.${column.sqliteColumn} -> ${postgresTable}.${column.postgresColumn} ` +
|
|
`(length=${column.actualLength}, max=${column.maxLength})`
|
|
)
|
|
})
|
|
}
|
|
|
|
throw rowError
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(`[migrate] ${sqliteTable}: ${rows.length} rows`)
|
|
}
|
|
|
|
if (!DRY_RUN) {
|
|
await pg.query('SET session_replication_role = DEFAULT')
|
|
await pg.query('COMMIT')
|
|
console.log('[migrate] migration transaction committed')
|
|
}
|
|
|
|
const parity = []
|
|
for (const { sqliteTable, postgresTable } of tablesToMigrate) {
|
|
const sqliteCountRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) AS count FROM ${quoteIdent(sqliteTable)}`)
|
|
const pgCountResult = await pg.query(`SELECT COUNT(*)::bigint AS count FROM ${quoteIdent(PG_SCHEMA)}.${quoteIdent(postgresTable)}`)
|
|
parity.push({
|
|
table: sqliteTable,
|
|
sqliteCount: Number(sqliteCountRow.count || 0),
|
|
postgresCount: Number(pgCountResult.rows[0].count || 0)
|
|
})
|
|
}
|
|
|
|
const mismatches = parity.filter((row) => row.sqliteCount !== row.postgresCount)
|
|
if (mismatches.length) {
|
|
console.error('[migrate] row-count mismatches detected:')
|
|
mismatches.forEach((row) => {
|
|
console.error(`[migrate] ${row.table}: sqlite=${row.sqliteCount} postgres=${row.postgresCount}`)
|
|
})
|
|
process.exitCode = 2
|
|
} else {
|
|
console.log('[migrate] parity check passed for all migrated tables')
|
|
}
|
|
} finally {
|
|
sqliteDb.close()
|
|
await pg.end()
|
|
}
|
|
}
|
|
|
|
if (require.main === module) {
|
|
main().catch((error) => {
|
|
console.error('[migrate] failed:', error)
|
|
process.exit(1)
|
|
})
|
|
}
|
|
|
|
module.exports = {
|
|
normalizeJson,
|
|
isIntegerCompatible,
|
|
convertValue,
|
|
findOverlongVarcharValues,
|
|
findIntegerTypeIssues,
|
|
quoteIdent
|
|
}
|