Fix:Server crash when deleting library item #2031

server/scanner/AudioFileScanner.js

@@ -0,0 +1,6 @@
+class AudioFileScanner {
+  constructor() { }
+
+
+}
+module.exports = new AudioFileScanner()

server/scanner/LibraryItemScanData.js

@@ -0,0 +1,147 @@
+const packageJson = require('../../package.json')
+const { LogLevel } = require('../utils/constants')
+const LibraryItem = require('../models/LibraryItem')
+
+class LibraryItemScanData {
+  constructor(data) {
+    /** @type {string} */
+    this.libraryFolderId = data.libraryFolderId
+    /** @type {string} */
+    this.libraryId = data.libraryId
+    /** @type {string} */
+    this.ino = data.ino
+    /** @type {number} */
+    this.mtimeMs = data.mtimeMs
+    /** @type {number} */
+    this.ctimeMs = data.ctimeMs
+    /** @type {number} */
+    this.birthtimeMs = data.birthtimeMs
+    /** @type {string} */
+    this.path = data.path
+    /** @type {string} */
+    this.relPath = data.relPath
+    /** @type {boolean} */
+    this.isFile = data.isFile
+    /** @type {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} */
+    this.mediaMetadata = data.mediaMetadata
+    /** @type {import('../objects/files/LibraryFile')[]} */
+    this.libraryFiles = data.libraryFiles
+
+    // Set after check
+    /** @type {boolean} */
+    this.hasChanges
+    /** @type {boolean} */
+    this.hasPathChange
+    /** @type {LibraryItem.LibraryFileObject[]} */
+    this.libraryFilesRemoved
+    /** @type {LibraryItem.LibraryFileObject[]} */
+    this.libraryFilesAdded
+    /** @type {LibraryItem.LibraryFileObject[]} */
+    this.libraryFilesModified
+  }
+
+  /**
+   * 
+   * @param {LibraryItem} existingLibraryItem 
+   * @param {import('./LibraryScan')} libraryScan
+   */
+  async checkLibraryItemData(existingLibraryItem, libraryScan) {
+    const keysToCompare = ['libraryFolderId', 'ino', 'mtimeMs', 'ctimeMs', 'birthtimeMs', 'path', 'relPath', 'isFile']
+    this.hasChanges = false
+    this.hasPathChange = false
+    for (const key of keysToCompare) {
+      if (existingLibraryItem[key] !== this[key]) {
+        libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "${key}" changed from "${existingLibraryItem[key]}" to "${this[key]}"`)
+        existingLibraryItem[key] = this[key]
+        this.hasChanges = true
+
+        if (key === 'relPath') {
+          this.hasPathChange = true
+        }
+      }
+    }
+
+    this.libraryFilesRemoved = []
+    this.libraryFilesModified = []
+    let libraryFilesAdded = this.libraryFiles.map(lf => lf)
+
+    for (const existingLibraryFile of existingLibraryItem.libraryFiles) {
+      // Find matching library file using path first and fallback to using inode value
+      let matchingLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === existingLibraryFile.metadata.path)
+      if (!matchingLibraryFile) {
+        matchingLibraryFile = this.libraryFiles.find(lf => lf.ino === existingLibraryFile.ino)
+        if (matchingLibraryFile) {
+          libraryScan.addLog(LogLevel.INFO, `Library file with path "${existingLibraryFile.metadata.path}" not found, but found file with matching inode value "${existingLibraryFile.ino}" at path "${matchingLibraryFile.metadata.path}"`)
+        }
+      }
+
+      if (!matchingLibraryFile) { // Library file removed
+        libraryScan.addLog(LogLevel.INFO, `Library file "${existingLibraryFile.metadata.path}" was removed from library item "${existingLibraryItem.path}"`)
+        this.libraryFilesRemoved.push(existingLibraryFile)
+        existingLibraryItem.libraryFiles = existingLibraryItem.libraryFiles.filter(lf => lf !== existingLibraryFile)
+        this.hasChanges = true
+      } else {
+        libraryFilesAdded = libraryFilesAdded.filter(lf => lf !== matchingLibraryFile)
+        if (this.compareUpdateLibraryFile(existingLibraryItem.path, existingLibraryFile, matchingLibraryFile, libraryScan)) {
+          this.libraryFilesModified.push(existingLibraryFile)
+          this.hasChanges = true
+        }
+      }
+    }
+
+    // Log new library files found
+    if (libraryFilesAdded.length) {
+      this.hasChanges = true
+      for (const libraryFile of libraryFilesAdded) {
+        libraryScan.addLog(LogLevel.INFO, `New library file found with path "${libraryFile.metadata.path}" for library item "${existingLibraryItem.path}"`)
+        existingLibraryItem.libraryFiles.push(libraryFile.toJSON())
+      }
+    }
+
+    if (this.hasChanges) {
+      existingLibraryItem.lastScan = Date.now()
+      existingLibraryItem.lastScanVersion = packageJson.version
+      await existingLibraryItem.save()
+    } else {
+      libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.path}" is up-to-date`)
+    }
+
+    this.libraryFilesAdded = libraryFilesAdded
+  }
+
+  /**
+   * Update existing library file with scanned in library file data
+   * @param {string} libraryItemPath
+   * @param {LibraryItem.LibraryFileObject} existingLibraryFile 
+   * @param {import('../objects/files/LibraryFile')} scannedLibraryFile 
+   * @param {import('./LibraryScan')} libraryScan
+   * @returns {boolean} false if no changes
+   */
+  compareUpdateLibraryFile(libraryItemPath, existingLibraryFile, scannedLibraryFile, libraryScan) {
+    let hasChanges = false
+
+    if (existingLibraryFile.ino !== scannedLibraryFile.ino) {
+      existingLibraryFile.ino = scannedLibraryFile.ino
+      hasChanges = true
+    }
+
+    for (const key in existingLibraryFile.metadata) {
+      if (existingLibraryFile.metadata[key] !== scannedLibraryFile.metadata[key]) {
+        if (key !== 'path' && key !== 'relPath') {
+          libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.path}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)
+        } else {
+          libraryScan.addLog(LogLevel.DEBUG, `Library file for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)
+        }
+        existingLibraryFile.metadata[key] = scannedLibraryFile.metadata[key]
+        hasChanges = true
+      }
+    }
+
+    if (hasChanges) {
+      existingLibraryFile.updatedAt = Date.now()
+    }
+
+    return hasChanges
+  }
+}
+module.exports = LibraryItemScanData

server/scanner/LibraryScanner.js

@@ -0,0 +1,230 @@
+const Path = require('path')
+const packageJson = require('../../package.json')
+const Logger = require('../Logger')
+const SocketAuthority = require('../SocketAuthority')
+const Database = require('../Database')
+const fs = require('../libs/fsExtra')
+const fileUtils = require('../utils/fileUtils')
+const scanUtils = require('../utils/scandir')
+const { ScanResult, LogLevel } = require('../utils/constants')
+const LibraryItemScanData = require('./LibraryItemScanData')
+
+class LibraryScanner {
+  constructor(coverManager, taskManager) {
+    this.coverManager = coverManager
+    this.taskManager = taskManager
+
+    this.cancelLibraryScan = {}
+    this.librariesScanning = []
+  }
+
+  /**
+   * @param {string} libraryId 
+   * @returns {boolean}
+   */
+  isLibraryScanning(libraryId) {
+    return this.librariesScanning.some(ls => ls.id === libraryId)
+  }
+
+  /**
+   * 
+   * @param {import('../objects/Library')} library 
+   * @param {*} options 
+   */
+  async scan(library, options = {}) {
+    if (this.isLibraryScanning(library.id)) {
+      Logger.error(`[Scanner] Already scanning ${library.id}`)
+      return
+    }
+
+    if (!library.folders.length) {
+      Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`)
+      return
+    }
+
+    const scanOptions = new ScanOptions()
+    scanOptions.setData(options, Database.serverSettings)
+
+    const libraryScan = new LibraryScan()
+    libraryScan.setData(library, scanOptions)
+    libraryScan.verbose = true
+    this.librariesScanning.push(libraryScan.getScanEmitData)
+
+    SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
+
+    Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
+
+    const canceled = await this.scanLibrary(libraryScan)
+
+    if (canceled) {
+      Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
+      delete this.cancelLibraryScan[libraryScan.libraryId]
+    }
+
+    libraryScan.setComplete()
+
+    Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
+    this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
+
+    if (canceled && !libraryScan.totalResults) {
+      const emitData = libraryScan.getScanEmitData
+      emitData.results = null
+      SocketAuthority.emitter('scan_complete', emitData)
+      return
+    }
+
+    SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
+
+    if (libraryScan.totalResults) {
+      libraryScan.saveLog()
+    }
+  }
+
+  /**
+   * 
+   * @param {import('./LibraryScan')} libraryScan 
+   */
+  async scanLibrary(libraryScan) {
+    /** @type {LibraryItemScanData[]} */
+    let libraryItemDataFound = []
+
+    // Scan each library folder
+    for (let i = 0; i < libraryScan.folders.length; i++) {
+      const folder = libraryScan.folders[i]
+      const itemDataFoundInFolder = await this.scanFolder(libraryScan.library, folder)
+      libraryScan.addLog(LogLevel.INFO, `${itemDataFoundInFolder.length} item data found in folder "${folder.fullPath}"`)
+      libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder)
+    }
+
+    if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+
+    const existingLibraryItems = await Database.libraryItemModel.findAll({
+      where: {
+        libraryId: libraryScan.libraryId
+      },
+      attributes: ['id', 'mediaId', 'mediaType', 'path', 'relPath', 'ino', 'isMissing', 'mtime', 'ctime', 'birthtime', 'libraryFiles', 'libraryFolderId']
+    })
+
+    const libraryItemIdsMissing = []
+    for (const existingLibraryItem of existingLibraryItems) {
+      // First try to find matching library item with exact file path
+      let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path)
+      if (!libraryItemData) {
+        // Fallback to finding matching library item with matching inode value
+        libraryItemData = libraryItemDataFound.find(lid => lid.ino === existingLibraryItem.ino)
+        if (libraryItemData) {
+          libraryScan.addLog(LogLevel.INFO, `Library item with path "${existingLibraryItem.path}" was not found, but library item inode "${existingLibraryItem.ino}" was found at path "${libraryItemData.path}"`)
+        }
+      }
+
+      if (!libraryItemData) {
+        // Podcast folder can have no episodes and still be valid
+        if (libraryScan.libraryMediaType === 'podcast' && await fs.pathExists(existingLibraryItem.path)) {
+          libraryScan.addLog(LogLevel.INFO, `Library item "${existingLibraryItem.relPath}" folder exists but has no episodes`)
+        } else {
+          libraryScan.addLog(LogLevel.WARN, `Library Item "${existingLibraryItem.path}" (inode: ${existingLibraryItem.ino}) is missing`)
+          if (!existingLibraryItem.isMissing) {
+            libraryItemIdsMissing.push(existingLibraryItem.id)
+          }
+        }
+      } else {
+        await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)
+        if (libraryItemData.hasChanges) {
+          await this.rescanLibraryItem(existingLibraryItem, libraryItemData)
+        }
+      }
+    }
+
+    // Update missing library items
+    if (libraryItemIdsMissing.length) {
+      libraryScan.addLog(LogLevel.INFO, `Updating ${libraryItemIdsMissing.length} library items missing`)
+      await Database.libraryItemModel.update({
+        isMissing: true,
+        lastScan: Date.now(),
+        lastScanVersion: packageJson.version
+      }, {
+        where: {
+          id: libraryItemIdsMissing
+        }
+      })
+    }
+  }
+
+  /**
+   * Get scan data for library folder
+   * @param {import('../objects/Library')} library 
+   * @param {import('../objects/Folder')} folder 
+   * @returns {LibraryItemScanData[]}
+   */
+  async scanFolder(library, folder) {
+    const folderPath = fileUtils.filePathToPOSIX(folder.fullPath)
+
+    const pathExists = await fs.pathExists(folderPath)
+    if (!pathExists) {
+      Logger.error(`[scandir] Invalid folder path does not exist "${folderPath}"`)
+      return []
+    }
+
+    const fileItems = await fileUtils.recurseFiles(folderPath)
+    const libraryItemGrouping = scanUtils.groupFileItemsIntoLibraryItemDirs(library.mediaType, fileItems, library.settings.audiobooksOnly)
+
+    if (!Object.keys(libraryItemGrouping).length) {
+      Logger.error(`Root path has no media folders: ${folderPath}`)
+      return []
+    }
+
+    const items = []
+    for (const libraryItemPath in libraryItemGrouping) {
+      let isFile = false // item is not in a folder
+      let libraryItemData = null
+      let fileObjs = []
+      if (libraryItemPath === libraryItemGrouping[libraryItemPath]) {
+        // Media file in root only get title
+        libraryItemData = {
+          mediaMetadata: {
+            title: Path.basename(libraryItemPath, Path.extname(libraryItemPath))
+          },
+          path: Path.posix.join(folderPath, libraryItemPath),
+          relPath: libraryItemPath
+        }
+        fileObjs = await scanUtils.buildLibraryFile(folderPath, [libraryItemPath])
+        isFile = true
+      } else {
+        libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath)
+        fileObjs = await scanUtils.buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath])
+      }
+
+      const libraryItemFolderStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)
+
+      if (!libraryItemFolderStats.ino) {
+        Logger.warn(`[LibraryScanner] Library item folder "${libraryItemData.path}" has no inode value`)
+        continue
+      }
+
+      items.push(new LibraryItemScanData({
+        libraryFolderId: folder.id,
+        libraryId: folder.libraryId,
+        ino: libraryItemFolderStats.ino,
+        mtimeMs: libraryItemFolderStats.mtimeMs || 0,
+        ctimeMs: libraryItemFolderStats.ctimeMs || 0,
+        birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
+        path: libraryItemData.path,
+        relPath: libraryItemData.relPath,
+        isFile,
+        mediaMetadata: libraryItemData.mediaMetadata || null,
+        libraryFiles: fileObjs
+      }))
+    }
+    return items
+  }
+
+  /**
+   * 
+   * @param {import('../models/LibraryItem')} existingLibraryItem 
+   * @param {LibraryItemScanData} libraryItemData 
+   */
+  async rescanLibraryItem(existingLibraryItem, libraryItemData) {
+
+  }
+}
+module.exports = LibraryScanner

server/scanner/MediaFileScanner.js

@@ -4,13 +4,18 @@ const AudioFile = require('../objects/files/AudioFile')
 const VideoFile = require('../objects/files/VideoFile')
 
 const prober = require('../utils/prober')
-const toneProber = require('../utils/toneProber')
 const Logger = require('../Logger')
 const { LogLevel } = require('../utils/constants')
 
 class MediaFileScanner {
   constructor() { }
 
+  /**
+   * Get track and disc number from audio filename
+   * @param {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} mediaMetadataFromScan 
+   * @param {import('../objects/files/LibraryFile')} audioLibraryFile 
+   * @returns {{trackNumber:number, discNumber:number}}
+   */
   getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, audioLibraryFile) {
     const { title, author, series, publishedYear } = mediaMetadataFromScan
     const { filename, path } = audioLibraryFile.metadata
@@ -102,7 +107,12 @@ class MediaFileScanner {
     }
   }
 
-  // Returns array of { MediaFile, elapsed, averageScanDuration } from audio file scan objects
+  /**
+   * Returns array of { MediaFile, elapsed, averageScanDuration } from audio file scan objects
+   * @param {import('../objects/LibraryItem')} libraryItem 
+   * @param {import('../objects/files/LibraryFile')[]} mediaLibraryFiles 
+   * @returns {Promise<object>}
+   */
   async executeMediaFileScans(libraryItem, mediaLibraryFiles) {
     const mediaType = libraryItem.mediaType
 
@@ -206,11 +216,10 @@ class MediaFileScanner {
 
   /**
   * Scans media files for a library item and adds them as audio tracks and sets library item metadata
-  * @async
-  * @param {Array<LibraryFile>} mediaLibraryFiles - Media files for this library item
-  * @param {LibraryItem} libraryItem
-  * @param {LibraryScan} [libraryScan=null] - Optional when doing a library scan to use LibraryScan config/logs
-  * @return {Promise<Boolean>} True if any updates were made
+  * @param {import('../objects/files/LibraryFile')[]} mediaLibraryFiles
+  * @param {import('../objects/LibraryItem')} libraryItem
+  * @param {import('./LibraryScan')} [libraryScan=null] - Optional when doing a library scan to use LibraryScan config/logs
+  * @return {Promise<boolean>} True if any updates were made
   */
   async scanMediaFiles(mediaLibraryFiles, libraryItem, libraryScan = null) {
     const preferAudioMetadata = libraryScan ? !!libraryScan.preferAudioMetadata : !!global.ServerSettings.scannerPreferAudioMetadata

server/scanner/Scanner.js

@@ -695,7 +695,7 @@ class Scanner {
         Logger.debug(`[Scanner] Folder update for relative path "${itemDir}" is in library item "${existingLibraryItem.media.metadata.title}" - scan for updates`)
         itemGroupingResults[itemDir] = await this.scanLibraryItem(library, folder, existingLibraryItem)
         continue
-      } else if (library.settings.audiobooksOnly && !fileUpdateGroup[itemDir].some(checkFilepathIsAudioFile)) {
+      } else if (library.settings.audiobooksOnly && !fileUpdateGroup[itemDir].some?.(checkFilepathIsAudioFile)) {
         Logger.debug(`[Scanner] Folder update for relative path "${itemDir}" has no audio files`)
         continue
       }