Added support for custom metadata providers

WiP but already open to feedback

server/Database.js

@@ -132,6 +132,11 @@ class Database {
     return this.models.playbackSession
   }
 
+  /** @type {typeof import('./models/CustomMetadataProvider')} */
+  get customMetadataProviderModel() {
+    return this.models.customMetadataProvider
+  }
+
   /**
    * Check if db file exists
    * @returns {boolean}
@@ -245,6 +250,7 @@ class Database {
     require('./models/Feed').init(this.sequelize)
     require('./models/FeedEpisode').init(this.sequelize)
     require('./models/Setting').init(this.sequelize)
+    require('./models/CustomMetadataProvider').init(this.sequelize)
 
     return this.sequelize.sync({ force, alter: false })
   }
@@ -694,6 +700,45 @@ class Database {
     })
   }
 
+  /**
+   * Returns true if a custom provider with the given slug exists
+   * @param {string} providerSlug
+   * @return {boolean}
+   */
+  async doesCustomProviderExistBySlug(providerSlug) {
+    const id = providerSlug.split("custom-")[1]
+
+    if (!id) {
+      return false
+    }
+
+    return !!await this.customMetadataProviderModel.findByPk(id)
+  }
+
+  /**
+   * Removes a custom metadata provider
+   * @param {string} id
+   */
+  async removeCustomMetadataProviderById(id) {
+    // destroy metadta provider
+    await this.customMetadataProviderModel.destroy({
+      where: {
+        id,
+      }
+    })
+
+    const slug = `custom-${id}`;
+
+    // fallback libraries using it to google
+    await this.libraryModel.update({
+      provider: "google",
+    }, {
+      where: {
+        provider: slug,
+      }
+    });
+  }
+
   /**
    * Clean invalid records in database
    * Series should have atleast one Book

server/controllers/LibraryController.js

@@ -51,6 +51,11 @@ class LibraryController {
       }
     }
 
+    // Validate that the custom provider exists if given any
+    if (newLibraryPayload.provider && newLibraryPayload.provider.startsWith("custom-")) {
+      await Database.doesCustomProviderExistBySlug(newLibraryPayload.provider)
+    }
+
     const library = new Library()
 
     let currentLargestDisplayOrder = await Database.libraryModel.getMaxDisplayOrder()
@@ -175,6 +180,11 @@ class LibraryController {
       }
     }
 
+    // Validate that the custom provider exists if given any
+    if (req.body.provider && req.body.provider.startsWith("custom-")) {
+      await Database.doesCustomProviderExistBySlug(req.body.provider)
+    }
+
     const hasUpdates = library.update(req.body)
     // TODO: Should check if this is an update to folder paths or name only
     if (hasUpdates) {

server/controllers/MiscController.js

@@ -717,5 +717,95 @@ class MiscController {
     const stats = await adminStats.getStatsForYear(year)
     res.json(stats)
   }
+
+  /**
+   * GET: /api/custom-metadata-providers
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
+  async getCustomMetadataProviders(req, res) {
+    const providers = await Database.customMetadataProviderModel.findAll()
+
+    res.json({
+      providers: providers.map((p) => p.toUserJson()),
+    })
+  }
+
+  /**
+   * GET: /api/custom-metadata-providers/admin
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
+  async getAdminCustomMetadataProviders(req, res) {
+    if (!req.user.isAdminOrUp) {
+      Logger.error(`[MiscController] Non-admin user "${req.user.username}" attempted to get admin custom metadata providers`)
+      return res.sendStatus(403)
+    }
+
+    const providers = await Database.customMetadataProviderModel.findAll()
+
+    res.json({
+      providers,
+    })
+  }
+
+  /**
+   * PATCH: /api/custom-metadata-providers/admin
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
+  async addCustomMetadataProviders(req, res) {
+    if (!req.user.isAdminOrUp) {
+      Logger.error(`[MiscController] Non-admin user "${req.user.username}" attempted to get admin custom metadata providers`)
+      return res.sendStatus(403)
+    }
+
+    const { name, url, apiKey } = req.body;
+
+    if (!name || !url || !apiKey) {
+      return res.status(500).send(`Invalid patch data`)
+    }
+
+    const provider = await Database.customMetadataProviderModel.create({
+      name,
+      url,
+      apiKey,
+    })
+
+    SocketAuthority.adminEmitter('custom_metadata_provider_added', provider)
+
+    res.json({
+      provider,
+    })
+  }
+
+  /**
+   * DELETE: /api/custom-metadata-providers/admin/:id
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
+  async deleteCustomMetadataProviders(req, res) {
+    if (!req.user.isAdminOrUp) {
+      Logger.error(`[MiscController] Non-admin user "${req.user.username}" attempted to get admin custom metadata providers`)
+      return res.sendStatus(403)
+    }
+
+    const { id } = req.params;
+
+    if (!id) {
+      return res.status(500).send(`Invalid delete data`)
+    }
+
+    const provider = await Database.customMetadataProviderModel.findByPk(id);
+    await Database.removeCustomMetadataProviderById(id);
+
+    SocketAuthority.adminEmitter('custom_metadata_provider_removed', provider)
+
+    res.json({})
+  }
 }
 module.exports = new MiscController()

server/finders/BookFinder.js

@@ -5,6 +5,7 @@ const iTunes = require('../providers/iTunes')
 const Audnexus = require('../providers/Audnexus')
 const FantLab = require('../providers/FantLab')
 const AudiobookCovers = require('../providers/AudiobookCovers')
+const CustomProviderAdapter = require('../providers/CustomProviderAdapter')
 const Logger = require('../Logger')
 const { levenshteinDistance, escapeRegExp } = require('../utils/index')
 
@@ -17,6 +18,7 @@ class BookFinder {
     this.audnexus = new Audnexus()
     this.fantLab = new FantLab()
     this.audiobookCovers = new AudiobookCovers()
+    this.customProviderAdapter = new CustomProviderAdapter()
 
     this.providers = ['google', 'itunes', 'openlibrary', 'fantlab', 'audiobookcovers', 'audible', 'audible.ca', 'audible.uk', 'audible.au', 'audible.fr', 'audible.de', 'audible.jp', 'audible.it', 'audible.in', 'audible.es']
 
@@ -147,6 +149,13 @@ class BookFinder {
     return books
   }
 
+  async getCustomProviderResults(title, author, providerSlug) {
+    const books = await this.customProviderAdapter.search(title, author, providerSlug)
+    if (this.verbose) Logger.debug(`Custom provider '${providerSlug}' Search Results: ${books.length || 0}`)
+
+    return books
+  }
+
   static TitleCandidates = class {
 
     constructor(cleanAuthor) {
@@ -315,6 +324,11 @@ class BookFinder {
     const maxFuzzySearches = !isNaN(options.maxFuzzySearches) ? Number(options.maxFuzzySearches) : 5
     let numFuzzySearches = 0
 
+    // Custom providers are assumed to be correct
+    if (provider.startsWith("custom-")) {
+      return await this.getCustomProviderResults(title, author, provider)
+    }
+
     if (!title)
       return books
 
@@ -397,8 +411,7 @@ class BookFinder {
       books = await this.getFantLabResults(title, author)
     } else if (provider === 'audiobookcovers') {
       books = await this.getAudiobookCoversResults(title)
-    }
-    else {
+    } else {
       books = await this.getGoogleBooksResults(title, author)
     }
     return books

server/models/CustomMetadataProvider.js

@@ -0,0 +1,58 @@
+const { DataTypes, Model, Sequelize } = require('sequelize')
+
+class CustomMetadataProvider extends Model {
+  constructor(values, options) {
+    super(values, options)
+
+    /** @type {UUIDV4} */
+    this.id
+    /** @type {string} */
+    this.name
+    /** @type {string} */
+    this.url
+    /** @type {string} */
+    this.apiKey
+  }
+
+  getSlug() {
+    return `custom-${this.id}`
+  }
+
+  toUserJson() {
+    return {
+      name: this.name,
+      id: this.id,
+      slug: this.getSlug()
+    }
+  }
+
+  static findByPk(id) {
+    this.findOne({
+      where: {
+        id,
+      }
+    })
+  }
+
+  /**
+   * Initialize model
+   * @param {import('../Database').sequelize} sequelize 
+   */
+  static init(sequelize) {
+    super.init({
+      id: {
+        type: DataTypes.UUID,
+        defaultValue: DataTypes.UUIDV4,
+        primaryKey: true
+      },
+      name: DataTypes.STRING,
+      url: DataTypes.STRING,
+      apiKey: DataTypes.STRING
+    }, {
+      sequelize,
+      modelName: 'customMetadataProvider'
+    })
+  }
+}
+
+module.exports = CustomMetadataProvider

server/providers/CustomProviderAdapter.js

@@ -0,0 +1,76 @@
+const Database = require('../Database')
+const axios = require("axios");
+const Logger = require("../Logger");
+
+class CustomProviderAdapter {
+    constructor() {
+    }
+
+    async search(title, author, providerSlug) {
+        const providerId = providerSlug.split("custom-")[1]
+
+        console.log(providerId)
+        const provider = await Database.customMetadataProviderModel.findOne({
+            where: {
+                id: providerId,
+            }
+        });
+
+        if (!provider) {
+            throw new Error("Custom provider not found for the given id");
+        }
+
+        const matches = await axios.get(`${provider.url}/search?query=${encodeURIComponent(title)}${!!author ? `&author=${encodeURIComponent(author)}` : ""}`, {
+            headers: {
+                "Authorization": provider.apiKey,
+            },
+        }).then((res) => {
+            if (!res || !res.data || !Array.isArray(res.data.matches)) return null
+            return res.data.matches
+        }).catch(error => {
+            Logger.error('[CustomMetadataProvider] Search error', error)
+            return []
+        })
+
+        if (matches === null) {
+            throw new Error("Custom provider returned malformed response");
+        }
+
+        // re-map keys to throw out
+        return matches.map(({
+            title,
+            subtitle,
+            author,
+            narrator,
+            publisher,
+            published_year,
+            description,
+            cover,
+            isbn,
+            asin,
+            genres,
+            tags,
+            language,
+            duration,
+        }) => {
+            return {
+                title,
+                subtitle,
+                author,
+                narrator,
+                publisher,
+                publishedYear: published_year,
+                description,
+                cover,
+                isbn,
+                asin,
+                genres,
+                tags: tags.join(","),
+                language,
+                duration,
+            }
+        })
+    }
+}
+
+module.exports = CustomProviderAdapter

server/routers/ApiRouter.js

@@ -318,6 +318,10 @@ class ApiRouter {
     this.router.patch('/auth-settings', MiscController.updateAuthSettings.bind(this))
     this.router.post('/watcher/update', MiscController.updateWatchedPath.bind(this))
     this.router.get('/stats/year/:year', MiscController.getAdminStatsForYear.bind(this))
+    this.router.get('/custom-metadata-providers', MiscController.getCustomMetadataProviders.bind(this))
+    this.router.get('/custom-metadata-providers/admin', MiscController.getAdminCustomMetadataProviders.bind(this))
+    this.router.patch('/custom-metadata-providers/admin', MiscController.addCustomMetadataProviders.bind(this))
+    this.router.delete('/custom-metadata-providers/admin/:id', MiscController.deleteCustomMetadataProviders.bind(this))
   }
 
   async getDirectories(dir, relpath, excludedDirs, level = 0) {