mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2025-12-24 12:49:38 +00:00
New scanner updates with library scan data
This commit is contained in:
advplyr
parent
14128f3e29
commit
bf11d266dc
15 changed files with 428 additions and 135 deletions
|
|
@ -1,22 +1,106 @@
|
|||
const Path = require('path')
|
||||
|
||||
const AudioFile = require('../objects/AudioFile')
|
||||
const AudioProbeData = require('./AudioProbeData')
|
||||
|
||||
const prober = require('../utils/prober')
|
||||
const Logger = require('../Logger')
|
||||
const { msToTimestamp } = require('../utils')
|
||||
|
||||
class AudioFileScanner {
|
||||
constructor() { }
|
||||
|
||||
async scan(audioFileData, verbose = false) {
|
||||
getTrackNumberFromMeta(scanData) {
|
||||
return !isNaN(scanData.trackNumber) && scanData.trackNumber !== null ? Math.trunc(Number(scanData.trackNumber)) : null
|
||||
}
|
||||
|
||||
getTrackNumberFromFilename(bookScanData, filename) {
|
||||
const { title, author, series, publishYear } = bookScanData
|
||||
var partbasename = Path.basename(filename, Path.extname(filename))
|
||||
|
||||
// Remove title, author, series, and publishYear from filename if there
|
||||
if (title) partbasename = partbasename.replace(title, '')
|
||||
if (author) partbasename = partbasename.replace(author, '')
|
||||
if (series) partbasename = partbasename.replace(series, '')
|
||||
if (publishYear) partbasename = partbasename.replace(publishYear)
|
||||
|
||||
// Remove eg. "disc 1" from path
|
||||
partbasename = partbasename.replace(/\bdisc \d\d?\b/i, '')
|
||||
|
||||
// Remove "cd01" or "cd 01" from path
|
||||
partbasename = partbasename.replace(/\bcd ?\d\d?\b/i, '')
|
||||
|
||||
var numbersinpath = partbasename.match(/\d{1,4}/g)
|
||||
if (!numbersinpath) return null
|
||||
|
||||
var number = numbersinpath.length ? parseInt(numbersinpath[0]) : null
|
||||
return number
|
||||
}
|
||||
|
||||
getCdNumberFromFilename(bookScanData, filename) {
|
||||
const { title, author, series, publishYear } = bookScanData
|
||||
var partbasename = Path.basename(filename, Path.extname(filename))
|
||||
|
||||
// Remove title, author, series, and publishYear from filename if there
|
||||
if (title) partbasename = partbasename.replace(title, '')
|
||||
if (author) partbasename = partbasename.replace(author, '')
|
||||
if (series) partbasename = partbasename.replace(series, '')
|
||||
if (publishYear) partbasename = partbasename.replace(publishYear)
|
||||
|
||||
var cdNumber = null
|
||||
|
||||
var cdmatch = partbasename.match(/\b(disc|cd) ?(\d\d?)\b/i)
|
||||
if (cdmatch && cdmatch.length > 2 && cdmatch[2]) {
|
||||
if (!isNaN(cdmatch[2])) {
|
||||
cdNumber = Number(cdmatch[2])
|
||||
}
|
||||
}
|
||||
|
||||
return cdNumber
|
||||
}
|
||||
|
||||
getAverageScanDurationMs(results) {
|
||||
if (!results.length) return 0
|
||||
var total = 0
|
||||
for (let i = 0; i < results.length; i++) total += results[i].elapsed
|
||||
return Math.floor(total / results.length)
|
||||
}
|
||||
|
||||
async scan(audioFileData, bookScanData, verbose = false) {
|
||||
var probeStart = Date.now()
|
||||
// Logger.debug(`[AudioFileScanner] Start Probe ${audioFileData.fullPath}`)
|
||||
var probeData = await prober.probe2(audioFileData.fullPath, verbose)
|
||||
if (probeData.error) {
|
||||
Logger.error(`[AudioFileScanner] ${probeData.error} : "${audioFileData.fullPath}"`)
|
||||
return null
|
||||
}
|
||||
// Logger.debug(`[AudioFileScanner] Finished Probe ${audioFileData.fullPath} elapsed ${msToTimestamp(Date.now() - probeStart, true)}`)
|
||||
|
||||
var audioFile = new AudioFile()
|
||||
// TODO: Build audio file
|
||||
return audioFile
|
||||
audioFileData.trackNumFromMeta = this.getTrackNumberFromMeta(probeData)
|
||||
audioFileData.trackNumFromFilename = this.getTrackNumberFromFilename(bookScanData, audioFileData.filename)
|
||||
audioFileData.cdNumFromFilename = this.getCdNumberFromFilename(bookScanData, audioFileData.filename)
|
||||
audioFile.setDataFromProbe(audioFileData, probeData)
|
||||
return {
|
||||
audioFile,
|
||||
elapsed: Date.now() - probeStart
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Returns array of { AudioFile, elapsed } from audio file scan objects
|
||||
async scanAudioFiles(audioFileDataArray, bookScanData) {
|
||||
var proms = []
|
||||
for (let i = 0; i < audioFileDataArray.length; i++) {
|
||||
var prom = this.scan(audioFileDataArray[i], bookScanData)
|
||||
proms.push(prom)
|
||||
}
|
||||
var scanStart = Date.now()
|
||||
var results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr))
|
||||
return {
|
||||
audioFiles: results.map(r => r.audioFile),
|
||||
elapsed: Date.now() - scanStart,
|
||||
averageScanDuration: this.getAverageScanDurationMs(results)
|
||||
}
|
||||
}
|
||||
}
|
||||
module.exports = new AudioFileScanner()
|
||||
|
|
@ -34,7 +34,7 @@ class AudioProbeData {
|
|||
}
|
||||
|
||||
setData(data) {
|
||||
var audioStream = getDefaultAudioStream(data.audio_streams)
|
||||
var audioStream = this.getDefaultAudioStream(data.audio_streams)
|
||||
|
||||
this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false
|
||||
this.format = data.format
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
const Folder = require('../objects/Folder')
|
||||
const Constants = require('../utils/constants')
|
||||
|
||||
const { getId } = require('../utils/index')
|
||||
const { getId, secondsToTimestamp } = require('../utils/index')
|
||||
|
||||
class LibraryScan {
|
||||
constructor() {
|
||||
|
|
@ -13,22 +14,49 @@ class LibraryScan {
|
|||
|
||||
this.startedAt = null
|
||||
this.finishedAt = null
|
||||
this.elapsed = null
|
||||
|
||||
this.folderScans = []
|
||||
this.status = Constants.ScanStatus.NOTHING
|
||||
this.resultsMissing = 0
|
||||
this.resultsAdded = 0
|
||||
this.resultsUpdated = 0
|
||||
}
|
||||
|
||||
get _scanOptions() { return this.scanOptions || {} }
|
||||
get forceRescan() { return !!this._scanOptions.forceRescan }
|
||||
|
||||
get resultStats() {
|
||||
return `${this.resultsAdded} Added | ${this.resultsUpdated} Updated | ${this.resultsMissing} Missing`
|
||||
}
|
||||
get elapsedTimestamp() {
|
||||
return secondsToTimestamp(this.elapsed / 1000)
|
||||
}
|
||||
get getScanEmitData() {
|
||||
return {
|
||||
id: this.libraryId,
|
||||
name: this.libraryName,
|
||||
results: {
|
||||
added: this.resultsAdded,
|
||||
updated: this.resultsUpdated,
|
||||
missing: this.resultsMissing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setData(library, scanOptions) {
|
||||
this.id = getId('lscan')
|
||||
this.libraryId = library.id
|
||||
this.libraryName = library.name
|
||||
this.folders = library.folders.map(folder => Folder(folder.toJSON()))
|
||||
this.folders = library.folders.map(folder => new Folder(folder.toJSON()))
|
||||
|
||||
this.scanOptions = scanOptions
|
||||
|
||||
this.startedAt = Date.now()
|
||||
}
|
||||
|
||||
setComplete() {
|
||||
this.finishedAt = Date.now()
|
||||
this.elapsed = this.finishedAt - this.startedAt
|
||||
}
|
||||
}
|
||||
module.exports = LibraryScan
|
||||
|
|
@ -4,33 +4,35 @@ class ScanOptions {
|
|||
constructor(options) {
|
||||
this.forceRescan = false
|
||||
|
||||
this.metadataPrecedence = [
|
||||
{
|
||||
id: 'directory',
|
||||
include: true
|
||||
},
|
||||
{
|
||||
id: 'reader-desc-txt',
|
||||
include: true
|
||||
},
|
||||
{
|
||||
id: 'audio-file-metadata',
|
||||
include: true
|
||||
},
|
||||
{
|
||||
id: 'metadata-opf',
|
||||
include: true
|
||||
},
|
||||
{
|
||||
id: 'external-source',
|
||||
include: false
|
||||
}
|
||||
]
|
||||
// this.metadataPrecedence = [
|
||||
// {
|
||||
// id: 'directory',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'reader-desc-txt',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'audio-file-metadata',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'metadata-opf',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'external-source',
|
||||
// include: false
|
||||
// }
|
||||
// ]
|
||||
|
||||
// Server settings
|
||||
this.parseSubtitles = false
|
||||
this.findCovers = false
|
||||
this.coverDestination = CoverDestination.METADATA
|
||||
this.preferAudioMetadata = false
|
||||
this.preferOpfMetadata = false
|
||||
|
||||
if (options) {
|
||||
this.construct(options)
|
||||
|
|
@ -53,7 +55,9 @@ class ScanOptions {
|
|||
metadataPrecedence: this.metadataPrecedence,
|
||||
parseSubtitles: this.parseSubtitles,
|
||||
findCovers: this.findCovers,
|
||||
coverDestination: this.coverDestination
|
||||
coverDestination: this.coverDestination,
|
||||
preferAudioMetadata: this.preferAudioMetadata,
|
||||
preferOpfMetadata: this.preferOpfMetadata
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -63,6 +67,8 @@ class ScanOptions {
|
|||
this.parseSubtitles = !!serverSettings.scannerParseSubtitle
|
||||
this.findCovers = !!serverSettings.scannerFindCovers
|
||||
this.coverDestination = serverSettings.coverDestination
|
||||
this.preferAudioMetadata = serverSettings.scannerPreferAudioMetadata
|
||||
this.preferOpfMetadata = serverSettings.scannerPreferOpfMetadata
|
||||
}
|
||||
}
|
||||
module.exports = ScanOptions
|
||||
|
|
@ -6,8 +6,7 @@ const Logger = require('../Logger')
|
|||
const { version } = require('../../package.json')
|
||||
const audioFileScanner = require('../utils/audioFileScanner')
|
||||
const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir')
|
||||
const { comparePaths, getIno, getId } = require('../utils/index')
|
||||
const { secondsToTimestamp } = require('../utils/fileUtils')
|
||||
const { comparePaths, getIno, getId, msToTimestamp } = require('../utils/index')
|
||||
const { ScanResult, CoverDestination } = require('../utils/constants')
|
||||
|
||||
const AudioFileScanner = require('./AudioFileScanner')
|
||||
|
|
@ -33,6 +32,20 @@ class Scanner {
|
|||
this.bookFinder = new BookFinder()
|
||||
}
|
||||
|
||||
getCoverDirectory(audiobook) {
|
||||
if (this.db.serverSettings.coverDestination === CoverDestination.AUDIOBOOK) {
|
||||
return {
|
||||
fullPath: audiobook.fullPath,
|
||||
relPath: '/s/book/' + audiobook.id
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
fullPath: Path.posix.join(this.BookMetadataPath, audiobook.id),
|
||||
relPath: Path.posix.join('/metadata', 'books', audiobook.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async scan(libraryId, options = {}) {
|
||||
if (this.librariesScanning.includes(libraryId)) {
|
||||
Logger.error(`[Scanner] Already scanning ${libraryId}`)
|
||||
|
|
@ -53,14 +66,19 @@ class Scanner {
|
|||
|
||||
var libraryScan = new LibraryScan()
|
||||
libraryScan.setData(library, scanOptions)
|
||||
this.librariesScanning.push(libraryScan)
|
||||
|
||||
this.emitter('scan_start', libraryScan.getScanEmitData)
|
||||
|
||||
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
|
||||
|
||||
var results = await this.scanLibrary(libraryScan)
|
||||
await this.scanLibrary(libraryScan)
|
||||
|
||||
Logger.info(`[Scanner] Library scan ${libraryScan.id} complete`)
|
||||
libraryScan.setComplete()
|
||||
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp}. ${libraryScan.resultStats}`)
|
||||
|
||||
return results
|
||||
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
|
||||
this.emitter('scan_complete', libraryScan.getScanEmitData)
|
||||
}
|
||||
|
||||
async scanLibrary(libraryScan) {
|
||||
|
|
@ -77,9 +95,9 @@ class Scanner {
|
|||
|
||||
var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId)
|
||||
|
||||
const audiobooksToUpdate = []
|
||||
const audiobooksToRescan = []
|
||||
const newAudiobookData = []
|
||||
var audiobooksToUpdate = []
|
||||
var audiobookRescans = []
|
||||
var newAudiobookScans = []
|
||||
|
||||
// Check for existing & removed audiobooks
|
||||
for (let i = 0; i < audiobooksInLibrary.length; i++) {
|
||||
|
|
@ -87,21 +105,20 @@ class Scanner {
|
|||
var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path))
|
||||
if (!dataFound) {
|
||||
Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`)
|
||||
audiobook.isMissing = true
|
||||
audiobook.lastUpdate = Date.now()
|
||||
scanResults.missing++
|
||||
audiobook.setMissing()
|
||||
audiobooksToUpdate.push(audiobook)
|
||||
} else {
|
||||
var checkRes = audiobook.checkShouldRescan(dataFound)
|
||||
var checkRes = audiobook.checkScanData(dataFound)
|
||||
if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) {
|
||||
// existing audiobook has new files
|
||||
checkRes.audiobook = audiobook
|
||||
audiobooksToRescan.push(checkRes)
|
||||
checkRes.bookScanData = dataFound
|
||||
audiobookRescans.push(this.rescanAudiobook(checkRes, libraryScan))
|
||||
libraryScan.resultsMissing++
|
||||
} else if (checkRes.updated) {
|
||||
audiobooksToUpdate.push(audiobook)
|
||||
libraryScan.resultsUpdated++
|
||||
}
|
||||
|
||||
// Remove this abf
|
||||
audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino)
|
||||
}
|
||||
}
|
||||
|
|
@ -113,60 +130,108 @@ class Scanner {
|
|||
if (!hasEbook && !dataFound.audioFiles.length) {
|
||||
Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
|
||||
} else {
|
||||
newAudiobookData.push(dataFound)
|
||||
newAudiobookScans.push(this.scanNewAudiobook(dataFound, libraryScan))
|
||||
}
|
||||
}
|
||||
|
||||
var rescans = []
|
||||
for (let i = 0; i < audiobooksToRescan.length; i++) {
|
||||
var rescan = this.rescanAudiobook(audiobooksToRescan[i])
|
||||
rescans.push(rescan)
|
||||
if (audiobookRescans.length) {
|
||||
var updatedAudiobooks = (await Promise.all(audiobookRescans)).filter(ab => !!ab)
|
||||
if (updatedAudiobooks.length) {
|
||||
audiobooksToUpdate = audiobooksToUpdate.concat(updatedAudiobooks)
|
||||
libraryScan.resultsUpdated += updatedAudiobooks.length
|
||||
}
|
||||
}
|
||||
var newscans = []
|
||||
for (let i = 0; i < newAudiobookData.length; i++) {
|
||||
var newscan = this.scanNewAudiobook(newAudiobookData[i])
|
||||
newscans.push(newscan)
|
||||
if (audiobooksToUpdate.length) {
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" updating ${audiobooksToUpdate.length} books`)
|
||||
await this.db.updateEntities('audiobook', audiobooksToUpdate)
|
||||
}
|
||||
|
||||
var rescanResults = await Promise.all(rescans)
|
||||
|
||||
var newscanResults = await Promise.all(newscans)
|
||||
|
||||
// TODO: Return report
|
||||
return {
|
||||
updates: 0,
|
||||
additions: 0
|
||||
if (newAudiobookScans.length) {
|
||||
var newAudiobooks = (await Promise.all(newAudiobookScans)).filter(ab => !!ab)
|
||||
if (newAudiobooks.length) {
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" inserting ${newAudiobooks.length} books`)
|
||||
await this.db.insertEntities('audiobook', newAudiobooks)
|
||||
libraryScan.resultsAdded = newAudiobooks.length
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return scan result payload
|
||||
async rescanAudiobook(audiobookCheckData) {
|
||||
const { newAudioFileData, newOtherFileData, audiobook } = audiobookCheckData
|
||||
async rescanAudiobook(audiobookCheckData, libraryScan) {
|
||||
const { newAudioFileData, newOtherFileData, audiobook, bookScanData } = audiobookCheckData
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`)
|
||||
|
||||
if (newAudioFileData.length) {
|
||||
var newAudioFiles = await this.scanAudioFiles(newAudioFileData)
|
||||
// TODO: Update audiobook tracks
|
||||
var audioScanResult = await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData)
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobook.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
|
||||
if (audioScanResult.audioFiles.length) {
|
||||
var totalAudioFilesToInclude = audiobook.audioFilesToInclude.length + audioScanResult.audioFiles.length
|
||||
|
||||
// validate & add audio files to audiobook
|
||||
for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
|
||||
var newAF = audioScanResult.audioFiles[i]
|
||||
var trackIndex = newAF.validateTrackIndex(totalAudioFilesToInclude === 1)
|
||||
if (trackIndex !== null) {
|
||||
if (audiobook.checkHasTrackNum(trackIndex)) {
|
||||
newAF.setDuplicateTrackNumber(trackIndex)
|
||||
} else {
|
||||
newAF.index = trackIndex
|
||||
}
|
||||
}
|
||||
audiobook.addAudioFile(newAF)
|
||||
}
|
||||
|
||||
audiobook.rebuildTracks()
|
||||
}
|
||||
}
|
||||
if (newOtherFileData.length) {
|
||||
// TODO: Check other files
|
||||
}
|
||||
|
||||
return {
|
||||
updated: true
|
||||
await audiobook.syncOtherFiles(newOtherFileData, this.MetadataPath)
|
||||
}
|
||||
return audiobook
|
||||
}
|
||||
|
||||
async scanNewAudiobook(audiobookData) {
|
||||
// TODO: Return new audiobook
|
||||
return null
|
||||
}
|
||||
async scanNewAudiobook(audiobookData, libraryScan) {
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Scanning new "${audiobookData.path}"`)
|
||||
var audiobook = new Audiobook()
|
||||
audiobook.setData(audiobookData)
|
||||
|
||||
async scanAudioFiles(audioFileData) {
|
||||
var proms = []
|
||||
for (let i = 0; i < audioFileData.length; i++) {
|
||||
var prom = AudioFileScanner.scan(audioFileData[i])
|
||||
proms.push(prom)
|
||||
if (audiobookData.audioFiles.length) {
|
||||
var audioScanResult = await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData)
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobookData.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
|
||||
if (audioScanResult.audioFiles.length) {
|
||||
// validate & add audio files to audiobook
|
||||
for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
|
||||
var newAF = audioScanResult.audioFiles[i]
|
||||
var trackIndex = newAF.validateTrackIndex(audioScanResult.audioFiles.length === 1)
|
||||
if (trackIndex !== null) {
|
||||
if (audiobook.checkHasTrackNum(trackIndex)) {
|
||||
newAF.setDuplicateTrackNumber(trackIndex)
|
||||
} else {
|
||||
newAF.index = trackIndex
|
||||
}
|
||||
}
|
||||
audiobook.addAudioFile(newAF)
|
||||
}
|
||||
audiobook.rebuildTracks()
|
||||
} else if (!audiobook.ebooks.length) {
|
||||
// Audiobook has no ebooks and no valid audio tracks do not continue
|
||||
Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
return Promise.all(proms)
|
||||
|
||||
// Look for desc.txt and reader.txt and update
|
||||
await audiobook.saveDataFromTextFiles()
|
||||
|
||||
// Extract embedded cover art if cover is not already in directory
|
||||
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
|
||||
var outputCoverDirs = this.getCoverDirectory(audiobook)
|
||||
var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
|
||||
if (relativeDir) {
|
||||
Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
|
||||
}
|
||||
}
|
||||
|
||||
return audiobook
|
||||
}
|
||||
}
|
||||
module.exports = Scanner
|
||||
Loading…
Add table
Add a link
Reference in a new issue