mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2025-12-24 04:39:40 +00:00
Add:New scanner and scanner server settings
This commit is contained in:
advplyr
parent
bf11d266dc
commit
a5fc382cad
17 changed files with 681 additions and 176 deletions
|
|
@ -4,7 +4,7 @@ const AudioFile = require('../objects/AudioFile')
|
|||
|
||||
const prober = require('../utils/prober')
|
||||
const Logger = require('../Logger')
|
||||
const { msToTimestamp } = require('../utils')
|
||||
const { LogLevel } = require('../utils/constants')
|
||||
|
||||
class AudioFileScanner {
|
||||
constructor() { }
|
||||
|
|
@ -80,6 +80,9 @@ class AudioFileScanner {
|
|||
audioFileData.trackNumFromFilename = this.getTrackNumberFromFilename(bookScanData, audioFileData.filename)
|
||||
audioFileData.cdNumFromFilename = this.getCdNumberFromFilename(bookScanData, audioFileData.filename)
|
||||
audioFile.setDataFromProbe(audioFileData, probeData)
|
||||
if (audioFile.embeddedCoverArt) {
|
||||
|
||||
}
|
||||
return {
|
||||
audioFile,
|
||||
elapsed: Date.now() - probeStart
|
||||
|
|
@ -87,12 +90,11 @@ class AudioFileScanner {
|
|||
}
|
||||
|
||||
|
||||
// Returns array of { AudioFile, elapsed } from audio file scan objects
|
||||
async scanAudioFiles(audioFileDataArray, bookScanData) {
|
||||
// Returns array of { AudioFile, elapsed, averageScanDuration } from audio file scan objects
|
||||
async executeAudioFileScans(audioFileDataArray, bookScanData) {
|
||||
var proms = []
|
||||
for (let i = 0; i < audioFileDataArray.length; i++) {
|
||||
var prom = this.scan(audioFileDataArray[i], bookScanData)
|
||||
proms.push(prom)
|
||||
proms.push(this.scan(audioFileDataArray[i], bookScanData))
|
||||
}
|
||||
var scanStart = Date.now()
|
||||
var results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr))
|
||||
|
|
@ -102,5 +104,62 @@ class AudioFileScanner {
|
|||
averageScanDuration: this.getAverageScanDurationMs(results)
|
||||
}
|
||||
}
|
||||
|
||||
async scanAudioFiles(audioFileDataArray, bookScanData, audiobook, preferAudioMetadata, libraryScan = null) {
|
||||
var hasUpdated = false
|
||||
|
||||
var audioScanResult = await this.executeAudioFileScans(audioFileDataArray, bookScanData)
|
||||
if (audioScanResult.audioFiles.length) {
|
||||
if (libraryScan) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Book "${bookScanData.path}" Audio file scan took ${audioScanResult.elapsed}ms for ${audioScanResult.audioFiles.length} with average time of ${audioScanResult.averageScanDuration}ms`)
|
||||
}
|
||||
|
||||
var totalAudioFilesToInclude = audiobook.audioFilesToInclude.filter(af => !audioScanResult.audioFiles.find(_af => _af.ino === af.ino)).length + audioScanResult.audioFiles.length
|
||||
|
||||
// validate & add/update audio files to audiobook
|
||||
for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
|
||||
var newAF = audioScanResult.audioFiles[i]
|
||||
var existingAF = audiobook.getAudioFileByIno(newAF.ino)
|
||||
|
||||
var trackIndex = null
|
||||
if (totalAudioFilesToInclude === 1) { // Single track audiobooks
|
||||
trackIndex = 1
|
||||
} else if (existingAF && existingAF.manuallyVerified) { // manually verified audio files use existing index
|
||||
trackIndex = existingAF.index
|
||||
} else {
|
||||
trackIndex = newAF.validateTrackIndex()
|
||||
}
|
||||
|
||||
if (trackIndex !== null) {
|
||||
if (audiobook.checkHasTrackNum(trackIndex, newAF.ino)) {
|
||||
newAF.setDuplicateTrackNumber(trackIndex)
|
||||
} else {
|
||||
newAF.index = trackIndex
|
||||
}
|
||||
}
|
||||
if (existingAF) {
|
||||
if (audiobook.updateAudioFile(newAF)) {
|
||||
// console.log('update dauido file')
|
||||
hasUpdated = true
|
||||
}
|
||||
} else {
|
||||
audiobook.addAudioFile(newAF)
|
||||
// console.log('added auido file')
|
||||
hasUpdated = true
|
||||
}
|
||||
}
|
||||
|
||||
if (hasUpdated) {
|
||||
audiobook.rebuildTracks()
|
||||
}
|
||||
|
||||
// Set book details from audio file ID3 tags, optional prefer
|
||||
if (audiobook.setDetailsFromFileMetadata(preferAudioMetadata)) {
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
}
|
||||
return hasUpdated
|
||||
}
|
||||
}
|
||||
module.exports = new AudioFileScanner()
|
||||
|
|
@ -35,7 +35,6 @@ class AudioProbeData {
|
|||
|
||||
setData(data) {
|
||||
var audioStream = this.getDefaultAudioStream(data.audio_streams)
|
||||
|
||||
this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false
|
||||
this.format = data.format
|
||||
this.duration = data.duration
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
const Folder = require('../objects/Folder')
|
||||
const Constants = require('../utils/constants')
|
||||
const Path = require('path')
|
||||
const fs = require('fs-extra')
|
||||
const date = require('date-and-time')
|
||||
|
||||
const Logger = require('../Logger')
|
||||
const Folder = require('../objects/Folder')
|
||||
const { LogLevel } = require('../utils/constants')
|
||||
const { getId, secondsToTimestamp } = require('../utils/index')
|
||||
|
||||
class LibraryScan {
|
||||
|
|
@ -9,6 +13,7 @@ class LibraryScan {
|
|||
this.libraryId = null
|
||||
this.libraryName = null
|
||||
this.folders = null
|
||||
this.verbose = false
|
||||
|
||||
this.scanOptions = null
|
||||
|
||||
|
|
@ -16,14 +21,21 @@ class LibraryScan {
|
|||
this.finishedAt = null
|
||||
this.elapsed = null
|
||||
|
||||
this.status = Constants.ScanStatus.NOTHING
|
||||
this.resultsMissing = 0
|
||||
this.resultsAdded = 0
|
||||
this.resultsUpdated = 0
|
||||
|
||||
this.logs = []
|
||||
}
|
||||
|
||||
get _scanOptions() { return this.scanOptions || {} }
|
||||
get forceRescan() { return !!this._scanOptions.forceRescan }
|
||||
get preferAudioMetadata() { return !!this._scanOptions.preferAudioMetadata }
|
||||
get preferOpfMetadata() { return !!this._scanOptions.preferOpfMetadata }
|
||||
get findCovers() { return !!this._scanOptions.findCovers }
|
||||
get timestamp() {
|
||||
return (new Date()).toISOString()
|
||||
}
|
||||
|
||||
get resultStats() {
|
||||
return `${this.resultsAdded} Added | ${this.resultsUpdated} Updated | ${this.resultsMissing} Missing`
|
||||
|
|
@ -42,6 +54,28 @@ class LibraryScan {
|
|||
}
|
||||
}
|
||||
}
|
||||
get totalResults() {
|
||||
return this.resultsAdded + this.resultsUpdated + this.resultsMissing
|
||||
}
|
||||
get getLogFilename() {
|
||||
return date.format(new Date(), 'YYYY-MM-DD') + '_' + this.id + '.txt'
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
return {
|
||||
id: this.id,
|
||||
libraryId: this.libraryId,
|
||||
libraryName: this.libraryName,
|
||||
folders: this.folders.map(f => f.toJSON()),
|
||||
scanOptions: this.scanOptions.toJSON(),
|
||||
startedAt: this.startedAt,
|
||||
finishedAt: this.finishedAt,
|
||||
elapsed: this.elapsed,
|
||||
resultsAdded: this.resultsAdded,
|
||||
resultsUpdated: this.resultsUpdated,
|
||||
resultsMissing: this.resultsMissing
|
||||
}
|
||||
}
|
||||
|
||||
setData(library, scanOptions) {
|
||||
this.id = getId('lscan')
|
||||
|
|
@ -58,5 +92,39 @@ class LibraryScan {
|
|||
this.finishedAt = Date.now()
|
||||
this.elapsed = this.finishedAt - this.startedAt
|
||||
}
|
||||
|
||||
getLogLevelString(level) {
|
||||
for (const key in LogLevel) {
|
||||
if (LogLevel[key] === level) {
|
||||
return key
|
||||
}
|
||||
}
|
||||
return 'UNKNOWN'
|
||||
}
|
||||
|
||||
addLog(level, ...args) {
|
||||
const logObj = {
|
||||
timestamp: this.timestamp,
|
||||
message: args.join(' '),
|
||||
levelName: this.getLogLevelString(level),
|
||||
level
|
||||
}
|
||||
|
||||
if (this.verbose) {
|
||||
Logger.debug(`[LibraryScan] "${this.libraryName}":`, args)
|
||||
}
|
||||
this.logs.push(logObj)
|
||||
}
|
||||
|
||||
async saveLog(logDir) {
|
||||
await fs.ensureDir(logDir)
|
||||
var outputPath = Path.join(logDir, this.getLogFilename)
|
||||
var logLines = [JSON.stringify(this.toJSON())]
|
||||
this.logs.forEach(l => {
|
||||
logLines.push(JSON.stringify(l))
|
||||
})
|
||||
await fs.writeFile(outputPath, logLines.join('\n') + '\n')
|
||||
Logger.info(`[LibraryScan] Scan log saved "${outputPath}"`)
|
||||
}
|
||||
}
|
||||
module.exports = LibraryScan
|
||||
|
|
@ -4,29 +4,6 @@ class ScanOptions {
|
|||
constructor(options) {
|
||||
this.forceRescan = false
|
||||
|
||||
// this.metadataPrecedence = [
|
||||
// {
|
||||
// id: 'directory',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'reader-desc-txt',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'audio-file-metadata',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'metadata-opf',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'external-source',
|
||||
// include: false
|
||||
// }
|
||||
// ]
|
||||
|
||||
// Server settings
|
||||
this.parseSubtitles = false
|
||||
this.findCovers = false
|
||||
|
|
|
|||
|
|
@ -4,10 +4,9 @@ const Path = require('path')
|
|||
// Utils
|
||||
const Logger = require('../Logger')
|
||||
const { version } = require('../../package.json')
|
||||
const audioFileScanner = require('../utils/audioFileScanner')
|
||||
const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir')
|
||||
const { comparePaths, getIno, getId, msToTimestamp } = require('../utils/index')
|
||||
const { ScanResult, CoverDestination } = require('../utils/constants')
|
||||
const { ScanResult, CoverDestination, LogLevel } = require('../utils/constants')
|
||||
|
||||
const AudioFileScanner = require('./AudioFileScanner')
|
||||
const BookFinder = require('../BookFinder')
|
||||
|
|
@ -20,6 +19,8 @@ class Scanner {
|
|||
this.AudiobookPath = AUDIOBOOK_PATH
|
||||
this.MetadataPath = METADATA_PATH
|
||||
this.BookMetadataPath = Path.posix.join(this.MetadataPath.replace(/\\/g, '/'), 'books')
|
||||
var LogDirPath = Path.join(this.MetadataPath, 'logs')
|
||||
this.ScanLogPath = Path.join(LogDirPath, 'scans')
|
||||
|
||||
this.db = db
|
||||
this.coverController = coverController
|
||||
|
|
@ -46,8 +47,82 @@ class Scanner {
|
|||
}
|
||||
}
|
||||
|
||||
isLibraryScanning(libraryId) {
|
||||
return this.librariesScanning.find(ls => ls.id === libraryId)
|
||||
}
|
||||
|
||||
async scanAudiobookById(audiobookId) {
|
||||
var audiobook = this.db.audiobooks.find(ab => ab.id === audiobookId)
|
||||
if (!audiobook) {
|
||||
Logger.error(`[Scanner] Scan audiobook by id not found ${audiobookId}`)
|
||||
return ScanResult.NOTHING
|
||||
}
|
||||
const library = this.db.libraries.find(lib => lib.id === audiobook.libraryId)
|
||||
if (!library) {
|
||||
Logger.error(`[Scanner] Scan audiobook by id library not found "${audiobook.libraryId}"`)
|
||||
return ScanResult.NOTHING
|
||||
}
|
||||
const folder = library.folders.find(f => f.id === audiobook.folderId)
|
||||
if (!folder) {
|
||||
Logger.error(`[Scanner] Scan audiobook by id folder not found "${audiobook.folderId}" in library "${library.name}"`)
|
||||
return ScanResult.NOTHING
|
||||
}
|
||||
Logger.info(`[Scanner] Scanning Audiobook "${audiobook.title}"`)
|
||||
return this.scanAudiobook(folder, audiobook)
|
||||
}
|
||||
|
||||
async scanAudiobook(folder, audiobook) {
|
||||
var audiobookData = await getAudiobookFileData(folder, audiobook.fullPath, this.db.serverSettings)
|
||||
if (!audiobookData) {
|
||||
return ScanResult.NOTHING
|
||||
}
|
||||
var hasUpdated = false
|
||||
|
||||
var checkRes = audiobook.checkScanData(audiobookData, version)
|
||||
if (checkRes.updated) hasUpdated = true
|
||||
|
||||
// Sync other files first so that local images are used as cover art
|
||||
// TODO: Cleanup other file sync
|
||||
var allOtherFiles = checkRes.newOtherFileData.concat(audiobook._otherFiles)
|
||||
if (await audiobook.syncOtherFiles(allOtherFiles, this.MetadataPath, this.db.serverSettings.scannerPreferOpfMetadata)) {
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
// Scan all audio files
|
||||
if (audiobookData.audioFiles.length) {
|
||||
if (await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData, audiobook, this.db.serverSettings.scannerPreferAudioMetadata)) {
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
// Extract embedded cover art if cover is not already in directory
|
||||
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
|
||||
var outputCoverDirs = this.getCoverDirectory(audiobook)
|
||||
var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
|
||||
if (relativeDir) {
|
||||
Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
|
||||
hasUpdated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { // Audiobook is invalid
|
||||
audiobook.setInvalid()
|
||||
hasUpdated = true
|
||||
} else if (audiobook.isInvalid) {
|
||||
audiobook.isInvalid = false
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
if (hasUpdated) {
|
||||
this.emitter('audiobook_updated', audiobook.toJSONExpanded())
|
||||
await this.db.updateEntity('audiobook', audiobook)
|
||||
return ScanResult.UPDATED
|
||||
}
|
||||
return ScanResult.UPTODATE
|
||||
}
|
||||
|
||||
async scan(libraryId, options = {}) {
|
||||
if (this.librariesScanning.includes(libraryId)) {
|
||||
if (this.isLibraryScanning(libraryId)) {
|
||||
Logger.error(`[Scanner] Already scanning ${libraryId}`)
|
||||
return
|
||||
}
|
||||
|
|
@ -66,172 +141,380 @@ class Scanner {
|
|||
|
||||
var libraryScan = new LibraryScan()
|
||||
libraryScan.setData(library, scanOptions)
|
||||
this.librariesScanning.push(libraryScan)
|
||||
libraryScan.verbose = false
|
||||
this.librariesScanning.push(libraryScan.getScanEmitData)
|
||||
|
||||
this.emitter('scan_start', libraryScan.getScanEmitData)
|
||||
|
||||
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
|
||||
|
||||
await this.scanLibrary(libraryScan)
|
||||
var canceled = await this.scanLibrary(libraryScan)
|
||||
|
||||
if (canceled) {
|
||||
Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
|
||||
delete this.cancelLibraryScan[libraryScan.libraryId]
|
||||
}
|
||||
|
||||
libraryScan.setComplete()
|
||||
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp}. ${libraryScan.resultStats}`)
|
||||
|
||||
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
|
||||
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
|
||||
|
||||
if (canceled && !libraryScan.totalResults) {
|
||||
var emitData = libraryScan.getScanEmitData
|
||||
emitData.results = null
|
||||
this.emitter('scan_complete', emitData)
|
||||
return
|
||||
}
|
||||
|
||||
this.emitter('scan_complete', libraryScan.getScanEmitData)
|
||||
|
||||
if (libraryScan.totalResults) {
|
||||
libraryScan.saveLog(this.ScanLogPath)
|
||||
}
|
||||
}
|
||||
|
||||
async scanLibrary(libraryScan) {
|
||||
var audiobookDataFound = []
|
||||
|
||||
// Scan each library
|
||||
for (let i = 0; i < libraryScan.folders.length; i++) {
|
||||
var folder = libraryScan.folders[i]
|
||||
var abDataFoundInFolder = await scanRootDir(folder, this.db.serverSettings)
|
||||
Logger.debug(`[Scanner] ${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`)
|
||||
libraryScan.addLog(LogLevel.INFO, `${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`)
|
||||
audiobookDataFound = audiobookDataFound.concat(abDataFoundInFolder)
|
||||
}
|
||||
|
||||
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
||||
|
||||
// Remove audiobooks with no inode
|
||||
audiobookDataFound = audiobookDataFound.filter(abd => abd.ino)
|
||||
|
||||
var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId)
|
||||
|
||||
const NumScansPerChunk = 25
|
||||
const audiobooksToUpdateChunks = []
|
||||
const audiobookDataToRescanChunks = []
|
||||
const newAudiobookDataToScanChunks = []
|
||||
var audiobooksToUpdate = []
|
||||
var audiobookRescans = []
|
||||
var newAudiobookScans = []
|
||||
var audiobookDataToRescan = []
|
||||
var newAudiobookDataToScan = []
|
||||
var audiobooksToFindCovers = []
|
||||
|
||||
// Check for existing & removed audiobooks
|
||||
for (let i = 0; i < audiobooksInLibrary.length; i++) {
|
||||
var audiobook = audiobooksInLibrary[i]
|
||||
var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path))
|
||||
if (!dataFound) {
|
||||
Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`)
|
||||
libraryScan.addLog(LogLevel.WARN, `Audiobook "${audiobook.title}" is missing`)
|
||||
libraryScan.resultsMissing++
|
||||
audiobook.setMissing()
|
||||
audiobooksToUpdate.push(audiobook)
|
||||
if (audiobooksToUpdate.length === NumScansPerChunk) {
|
||||
audiobooksToUpdateChunks.push(audiobooksToUpdate)
|
||||
audiobooksToUpdate = []
|
||||
}
|
||||
} else {
|
||||
var checkRes = audiobook.checkScanData(dataFound)
|
||||
if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) {
|
||||
// existing audiobook has new files
|
||||
var checkRes = audiobook.checkScanData(dataFound, version)
|
||||
if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) { // Audiobook has new files
|
||||
checkRes.audiobook = audiobook
|
||||
checkRes.bookScanData = dataFound
|
||||
audiobookRescans.push(this.rescanAudiobook(checkRes, libraryScan))
|
||||
libraryScan.resultsMissing++
|
||||
} else if (checkRes.updated) {
|
||||
audiobooksToUpdate.push(audiobook)
|
||||
audiobookDataToRescan.push(checkRes)
|
||||
if (audiobookDataToRescan.length === NumScansPerChunk) {
|
||||
audiobookDataToRescanChunks.push(audiobookDataToRescan)
|
||||
audiobookDataToRescan = []
|
||||
}
|
||||
} else if (libraryScan.findCovers && audiobook.book.shouldSearchForCover) {
|
||||
libraryScan.resultsUpdated++
|
||||
audiobooksToFindCovers.push(audiobook)
|
||||
audiobooksToUpdate.push(audiobook)
|
||||
if (audiobooksToUpdate.length === NumScansPerChunk) {
|
||||
audiobooksToUpdateChunks.push(audiobooksToUpdate)
|
||||
audiobooksToUpdate = []
|
||||
}
|
||||
} else if (checkRes.updated) { // Updated but no scan required
|
||||
libraryScan.resultsUpdated++
|
||||
audiobooksToUpdate.push(audiobook)
|
||||
if (audiobooksToUpdate.length === NumScansPerChunk) {
|
||||
audiobooksToUpdateChunks.push(audiobooksToUpdate)
|
||||
audiobooksToUpdate = []
|
||||
}
|
||||
}
|
||||
audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino)
|
||||
}
|
||||
}
|
||||
if (audiobooksToUpdate.length) audiobooksToUpdateChunks.push(audiobooksToUpdate)
|
||||
if (audiobookDataToRescan.length) audiobookDataToRescanChunks.push(audiobookDataToRescan)
|
||||
|
||||
// Potential NEW Audiobooks
|
||||
for (let i = 0; i < audiobookDataFound.length; i++) {
|
||||
var dataFound = audiobookDataFound[i]
|
||||
var hasEbook = dataFound.otherFiles.find(otherFile => otherFile.filetype === 'ebook')
|
||||
if (!hasEbook && !dataFound.audioFiles.length) {
|
||||
Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
|
||||
libraryScan.addLog(LogLevel.WARN, `Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
|
||||
} else {
|
||||
newAudiobookScans.push(this.scanNewAudiobook(dataFound, libraryScan))
|
||||
newAudiobookDataToScan.push(dataFound)
|
||||
if (newAudiobookDataToScan.length === NumScansPerChunk) {
|
||||
newAudiobookDataToScanChunks.push(newAudiobookDataToScan)
|
||||
newAudiobookDataToScan = []
|
||||
}
|
||||
}
|
||||
}
|
||||
if (newAudiobookDataToScan.length) newAudiobookDataToScanChunks.push(newAudiobookDataToScan)
|
||||
|
||||
// console.log('Num chunks to update', audiobooksToUpdateChunks.length)
|
||||
// console.log('Num chunks to rescan', audiobookDataToRescanChunks.length)
|
||||
// console.log('Num chunks to new scan', newAudiobookDataToScanChunks.length)
|
||||
|
||||
// Audiobooks not requiring a scan but require a search for cover
|
||||
for (let i = 0; i < audiobooksToFindCovers.length; i++) {
|
||||
var audiobook = audiobooksToFindCovers[i]
|
||||
var updatedCover = await this.searchForCover(audiobook, libraryScan)
|
||||
audiobook.book.updateLastCoverSearch(updatedCover)
|
||||
}
|
||||
|
||||
if (audiobookRescans.length) {
|
||||
var updatedAudiobooks = (await Promise.all(audiobookRescans)).filter(ab => !!ab)
|
||||
if (updatedAudiobooks.length) {
|
||||
audiobooksToUpdate = audiobooksToUpdate.concat(updatedAudiobooks)
|
||||
libraryScan.resultsUpdated += updatedAudiobooks.length
|
||||
}
|
||||
for (let i = 0; i < audiobooksToUpdateChunks.length; i++) {
|
||||
await this.updateAudiobooksChunk(audiobooksToUpdateChunks[i])
|
||||
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
||||
// console.log('Update chunk done', i, 'of', audiobooksToUpdateChunks.length)
|
||||
}
|
||||
if (audiobooksToUpdate.length) {
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" updating ${audiobooksToUpdate.length} books`)
|
||||
await this.db.updateEntities('audiobook', audiobooksToUpdate)
|
||||
for (let i = 0; i < audiobookDataToRescanChunks.length; i++) {
|
||||
await this.rescanAudiobookDataChunk(audiobookDataToRescanChunks[i], libraryScan)
|
||||
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
||||
// console.log('Rescan chunk done', i, 'of', audiobookDataToRescanChunks.length)
|
||||
}
|
||||
for (let i = 0; i < newAudiobookDataToScanChunks.length; i++) {
|
||||
await this.scanNewAudiobookDataChunk(newAudiobookDataToScanChunks[i], libraryScan)
|
||||
// console.log('New scan chunk done', i, 'of', newAudiobookDataToScanChunks.length)
|
||||
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
||||
}
|
||||
}
|
||||
|
||||
if (newAudiobookScans.length) {
|
||||
var newAudiobooks = (await Promise.all(newAudiobookScans)).filter(ab => !!ab)
|
||||
if (newAudiobooks.length) {
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" inserting ${newAudiobooks.length} books`)
|
||||
await this.db.insertEntities('audiobook', newAudiobooks)
|
||||
libraryScan.resultsAdded = newAudiobooks.length
|
||||
}
|
||||
}
|
||||
async updateAudiobooksChunk(audiobooksToUpdate) {
|
||||
await this.db.updateEntities('audiobook', audiobooksToUpdate)
|
||||
this.emitter('audiobooks_updated', audiobooksToUpdate.map(ab => ab.toJSONExpanded()))
|
||||
}
|
||||
|
||||
async rescanAudiobookDataChunk(audiobookDataToRescan, libraryScan) {
|
||||
var audiobooksUpdated = await Promise.all(audiobookDataToRescan.map((abd) => {
|
||||
return this.rescanAudiobook(abd, libraryScan)
|
||||
}))
|
||||
audiobooksUpdated = audiobooksUpdated.filter(ab => ab) // Filter out nulls
|
||||
libraryScan.resultsUpdated += audiobooksUpdated.length
|
||||
await this.db.updateEntities('audiobook', audiobooksUpdated)
|
||||
this.emitter('audiobooks_updated', audiobooksUpdated.map(ab => ab.toJSONExpanded()))
|
||||
}
|
||||
|
||||
async scanNewAudiobookDataChunk(newAudiobookDataToScan, libraryScan) {
|
||||
var newAudiobooks = await Promise.all(newAudiobookDataToScan.map((abd) => {
|
||||
return this.scanNewAudiobook(abd, libraryScan.preferAudioMetadata, libraryScan.preferOpfMetadata, libraryScan.findCovers, libraryScan)
|
||||
}))
|
||||
newAudiobooks = newAudiobooks.filter(ab => ab) // Filter out nulls
|
||||
libraryScan.resultsAdded += newAudiobooks.length
|
||||
await this.db.insertEntities('audiobook', newAudiobooks)
|
||||
this.emitter('audiobooks_added', newAudiobooks.map(ab => ab.toJSONExpanded()))
|
||||
}
|
||||
|
||||
async rescanAudiobook(audiobookCheckData, libraryScan) {
|
||||
const { newAudioFileData, newOtherFileData, audiobook, bookScanData } = audiobookCheckData
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`)
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`)
|
||||
|
||||
// Sync other files first to use local images as cover before extracting audio file cover
|
||||
if (newOtherFileData.length) {
|
||||
// TODO: Cleanup other file sync
|
||||
var allOtherFiles = newOtherFileData.concat(audiobook._otherFiles)
|
||||
await audiobook.syncOtherFiles(allOtherFiles, this.MetadataPath, libraryScan.preferOpfMetadata)
|
||||
}
|
||||
|
||||
if (newAudioFileData.length) {
|
||||
var audioScanResult = await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData)
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobook.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
|
||||
if (audioScanResult.audioFiles.length) {
|
||||
var totalAudioFilesToInclude = audiobook.audioFilesToInclude.length + audioScanResult.audioFiles.length
|
||||
await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData, audiobook, libraryScan.preferAudioMetadata, libraryScan)
|
||||
|
||||
// validate & add audio files to audiobook
|
||||
for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
|
||||
var newAF = audioScanResult.audioFiles[i]
|
||||
var trackIndex = newAF.validateTrackIndex(totalAudioFilesToInclude === 1)
|
||||
if (trackIndex !== null) {
|
||||
if (audiobook.checkHasTrackNum(trackIndex)) {
|
||||
newAF.setDuplicateTrackNumber(trackIndex)
|
||||
} else {
|
||||
newAF.index = trackIndex
|
||||
}
|
||||
}
|
||||
audiobook.addAudioFile(newAF)
|
||||
// Extract embedded cover art if cover is not already in directory
|
||||
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
|
||||
var outputCoverDirs = this.getCoverDirectory(audiobook)
|
||||
var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
|
||||
if (relativeDir) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${relativeDir}"`)
|
||||
}
|
||||
|
||||
audiobook.rebuildTracks()
|
||||
}
|
||||
}
|
||||
if (newOtherFileData.length) {
|
||||
await audiobook.syncOtherFiles(newOtherFileData, this.MetadataPath)
|
||||
|
||||
if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { // Audiobook is invalid
|
||||
audiobook.setInvalid()
|
||||
} else if (audiobook.isInvalid) {
|
||||
audiobook.isInvalid = false
|
||||
}
|
||||
|
||||
// Scan for cover if enabled and has no cover
|
||||
if (audiobook && libraryScan.findCovers && !audiobook.cover && audiobook.book.shouldSearchForCover) {
|
||||
var updatedCover = await this.searchForCover(audiobook, libraryScan)
|
||||
audiobook.book.updateLastCoverSearch(updatedCover)
|
||||
}
|
||||
|
||||
return audiobook
|
||||
}
|
||||
|
||||
async scanNewAudiobook(audiobookData, libraryScan) {
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Scanning new "${audiobookData.path}"`)
|
||||
async scanNewAudiobook(audiobookData, preferAudioMetadata, preferOpfMetadata, findCovers, libraryScan = null) {
|
||||
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new book "${audiobookData.path}"`)
|
||||
else Logger.debug(`[Scanner] Scanning new book "${audiobookData.path}"`)
|
||||
|
||||
var audiobook = new Audiobook()
|
||||
audiobook.setData(audiobookData)
|
||||
|
||||
if (audiobookData.audioFiles.length) {
|
||||
var audioScanResult = await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData)
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobookData.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
|
||||
if (audioScanResult.audioFiles.length) {
|
||||
// validate & add audio files to audiobook
|
||||
for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
|
||||
var newAF = audioScanResult.audioFiles[i]
|
||||
var trackIndex = newAF.validateTrackIndex(audioScanResult.audioFiles.length === 1)
|
||||
if (trackIndex !== null) {
|
||||
if (audiobook.checkHasTrackNum(trackIndex)) {
|
||||
newAF.setDuplicateTrackNumber(trackIndex)
|
||||
} else {
|
||||
newAF.index = trackIndex
|
||||
}
|
||||
}
|
||||
audiobook.addAudioFile(newAF)
|
||||
}
|
||||
audiobook.rebuildTracks()
|
||||
} else if (!audiobook.ebooks.length) {
|
||||
// Audiobook has no ebooks and no valid audio tracks do not continue
|
||||
Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`)
|
||||
return null
|
||||
}
|
||||
await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData, audiobook, preferAudioMetadata, libraryScan)
|
||||
}
|
||||
|
||||
if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) {
|
||||
// Audiobook has no ebooks and no valid audio tracks do not continue
|
||||
Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`)
|
||||
return null
|
||||
}
|
||||
|
||||
// Look for desc.txt and reader.txt and update
|
||||
await audiobook.saveDataFromTextFiles()
|
||||
await audiobook.saveDataFromTextFiles(preferOpfMetadata)
|
||||
|
||||
// Extract embedded cover art if cover is not already in directory
|
||||
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
|
||||
var outputCoverDirs = this.getCoverDirectory(audiobook)
|
||||
var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
|
||||
if (relativeDir) {
|
||||
Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
|
||||
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${relativeDir}"`)
|
||||
else Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
|
||||
}
|
||||
}
|
||||
|
||||
// Scan for cover if enabled and has no cover
|
||||
if (audiobook && findCovers && !audiobook.cover && audiobook.book.shouldSearchForCover) {
|
||||
var updatedCover = await this.searchForCover(audiobook, libraryScan)
|
||||
audiobook.book.updateLastCoverSearch(updatedCover)
|
||||
}
|
||||
|
||||
return audiobook
|
||||
}
|
||||
|
||||
getFileUpdatesGrouped(fileUpdates) {
|
||||
var folderGroups = {}
|
||||
fileUpdates.forEach((file) => {
|
||||
if (folderGroups[file.folderId]) {
|
||||
folderGroups[file.folderId].fileUpdates.push(file)
|
||||
} else {
|
||||
folderGroups[file.folderId] = {
|
||||
libraryId: file.libraryId,
|
||||
folderId: file.folderId,
|
||||
fileUpdates: [file]
|
||||
}
|
||||
}
|
||||
})
|
||||
return folderGroups
|
||||
}
|
||||
|
||||
async scanFilesChanged(fileUpdates) {
|
||||
if (!fileUpdates.length) return
|
||||
// files grouped by folder
|
||||
var folderGroups = this.getFileUpdatesGrouped(fileUpdates)
|
||||
|
||||
for (const folderId in folderGroups) {
|
||||
var libraryId = folderGroups[folderId].libraryId
|
||||
var library = this.db.libraries.find(lib => lib.id === libraryId)
|
||||
if (!library) {
|
||||
Logger.error(`[Scanner] Library not found in files changed ${libraryId}`)
|
||||
continue;
|
||||
}
|
||||
var folder = library.getFolderById(folderId)
|
||||
if (!folder) {
|
||||
Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`)
|
||||
continue;
|
||||
}
|
||||
var relFilePaths = folderGroups[folderId].fileUpdates.map(fileUpdate => fileUpdate.relPath)
|
||||
var fileUpdateBookGroup = groupFilesIntoAudiobookPaths(relFilePaths, true)
|
||||
var folderScanResults = await this.scanFolderUpdates(library, folder, fileUpdateBookGroup)
|
||||
Logger.debug(`[Scanner] Folder scan results`, folderScanResults)
|
||||
}
|
||||
}
|
||||
|
||||
async scanFolderUpdates(library, folder, fileUpdateBookGroup) {
|
||||
Logger.debug(`[Scanner] Scanning file update groups in folder "${folder.id}" of library "${library.name}"`)
|
||||
|
||||
var bookGroupingResults = {}
|
||||
for (const bookDir in fileUpdateBookGroup) {
|
||||
var fullPath = Path.posix.join(folder.fullPath.replace(/\\/g, '/'), bookDir)
|
||||
|
||||
// Check if book dir group is already an audiobook or in a subdir of an audiobook
|
||||
var existingAudiobook = this.db.audiobooks.find(ab => fullPath.startsWith(ab.fullPath))
|
||||
if (existingAudiobook) {
|
||||
|
||||
// Is the audiobook exactly - check if was deleted
|
||||
if (existingAudiobook.fullPath === fullPath) {
|
||||
var exists = await fs.pathExists(fullPath)
|
||||
if (!exists) {
|
||||
Logger.info(`[Scanner] Scanning file update group and audiobook was deleted "${existingAudiobook.title}" - marking as missing`)
|
||||
existingAudiobook.setMissing()
|
||||
await this.db.updateAudiobook(existingAudiobook)
|
||||
this.emitter('audiobook_updated', existingAudiobook.toJSONExpanded())
|
||||
|
||||
bookGroupingResults[bookDir] = ScanResult.REMOVED
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Scan audiobook for updates
|
||||
Logger.debug(`[Scanner] Folder update for relative path "${bookDir}" is in audiobook "${existingAudiobook.title}" - scan for updates`)
|
||||
bookGroupingResults[bookDir] = await this.scanAudiobook(folder, existingAudiobook)
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if an audiobook is a subdirectory of this dir
|
||||
var childAudiobook = this.db.audiobooks.find(ab => ab.fullPath.startsWith(fullPath))
|
||||
if (childAudiobook) {
|
||||
Logger.warn(`[Scanner] Files were modified in a parent directory of an audiobook "${childAudiobook.title}" - ignoring`)
|
||||
bookGroupingResults[bookDir] = ScanResult.NOTHING
|
||||
continue;
|
||||
}
|
||||
|
||||
Logger.debug(`[Scanner] Folder update group must be a new book "${bookDir}" in library "${library.name}"`)
|
||||
var newAudiobook = await this.scanPotentialNewAudiobook(folder, fullPath)
|
||||
if (newAudiobook) {
|
||||
await this.db.insertEntity('audiobook', newAudiobook)
|
||||
this.emitter('audiobook_added', newAudiobook.toJSONExpanded())
|
||||
}
|
||||
bookGroupingResults[bookDir] = newAudiobook ? ScanResult.ADDED : ScanResult.NOTHING
|
||||
}
|
||||
|
||||
return bookGroupingResults
|
||||
}
|
||||
|
||||
async scanPotentialNewAudiobook(folder, fullPath) {
|
||||
var audiobookData = await getAudiobookFileData(folder, fullPath, this.db.serverSettings)
|
||||
if (!audiobookData) return null
|
||||
var serverSettings = this.db.serverSettings
|
||||
return this.scanNewAudiobook(audiobookData, serverSettings.scannerPreferAudioMetadata, serverSettings.scannerPreferOpfMetadata, serverSettings.scannerFindCovers)
|
||||
}
|
||||
|
||||
async searchForCover(audiobook, libraryScan = null) {
|
||||
var options = {
|
||||
titleDistance: 2,
|
||||
authorDistance: 2
|
||||
}
|
||||
var results = await this.bookFinder.findCovers('google', audiobook.title, audiobook.authorFL, options)
|
||||
if (results.length) {
|
||||
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Found best cover for "${audiobook.title}"`)
|
||||
else Logger.debug(`[Scanner] Found best cover for "${audiobook.title}"`)
|
||||
|
||||
// If the first cover result fails, attempt to download the second
|
||||
for (let i = 0; i < results.length && i < 2; i++) {
|
||||
|
||||
// Downloads and updates the book cover
|
||||
var result = await this.coverController.downloadCoverFromUrl(audiobook, results[i])
|
||||
|
||||
if (result.error) {
|
||||
Logger.error(`[Scanner] Failed to download cover from url "${results[i]}" | Attempt ${i + 1}`, result.error)
|
||||
} else {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
module.exports = Scanner
|
||||
Loading…
Add table
Add a link
Reference in a new issue