Add infra to handle with feed healthy

This commit is contained in:
mfcar 2023-09-23 20:38:03 +01:00
parent 102c90c4e8
commit 2b3c3cd088
No known key found for this signature in database
6 changed files with 412 additions and 365 deletions

View file

@ -485,16 +485,16 @@ export default {
return this.$toast.error('Podcast does not have an RSS Feed') return this.$toast.error('Podcast does not have an RSS Feed')
} }
this.fetchingRSSFeed = true this.fetchingRSSFeed = true
var payload = await this.$axios.$post(`/api/podcasts/feed`, { rssFeed: this.mediaMetadata.feedUrl }).catch((error) => { var payload = await this.$axios.get(`/api/podcasts/${this.libraryItemId}/feed`).catch((error) => {
console.error('Failed to get feed', error) console.error('Failed to get feed', error)
this.$toast.error('Failed to get podcast feed') this.$toast.error('Failed to get podcast feed')
return null return null
}) })
this.fetchingRSSFeed = false this.fetchingRSSFeed = false
if (!payload) return if (!payload || !payload.data) return
console.log('Podcast feed', payload) console.log('Podcast feed', payload.data)
const podcastfeed = payload.podcast const podcastfeed = payload.data.podcast
if (!podcastfeed.episodes || !podcastfeed.episodes.length) { if (!podcastfeed.episodes || !podcastfeed.episodes.length) {
this.$toast.info('No episodes found in RSS feed') this.$toast.info('No episodes found in RSS feed')
return return

View file

@ -115,6 +115,19 @@ class PodcastController {
res.json({ podcast }) res.json({ podcast })
} }
async checkPodcastFeed(req, res) {
const libraryItem = req.libraryItem
const podcast = await getPodcastFeed(libraryItem.media.metadata.feedUrl)
if (!podcast) {
this.podcastManager.setFeedHealthStatus(libraryItem, false)
return res.status(404).send('Podcast RSS feed request failed or invalid response data')
}
this.podcastManager.setFeedHealthStatus(libraryItem, true)
res.json({ podcast })
}
async getFeedsFromOPMLText(req, res) { async getFeedsFromOPMLText(req, res) {
if (!req.body.opmlText) { if (!req.body.opmlText) {
return res.sendStatus(400) return res.sendStatus(400)

View file

@ -4,9 +4,9 @@ const Database = require('../Database')
const fs = require('../libs/fsExtra') const fs = require('../libs/fsExtra')
const { getPodcastFeed } = require('../utils/podcastUtils') const {getPodcastFeed} = require('../utils/podcastUtils')
const { removeFile, downloadFile } = require('../utils/fileUtils') const {removeFile, downloadFile} = require('../utils/fileUtils')
const { levenshteinDistance } = require('../utils/index') const {levenshteinDistance} = require('../utils/index')
const opmlParser = require('../utils/parsers/parseOPML') const opmlParser = require('../utils/parsers/parseOPML')
const opmlGenerator = require('../utils/generators/opmlGenerator') const opmlGenerator = require('../utils/generators/opmlGenerator')
const prober = require('../utils/prober') const prober = require('../utils/prober')
@ -19,374 +19,395 @@ const AudioFile = require('../objects/files/AudioFile')
const Task = require("../objects/Task") const Task = require("../objects/Task")
class PodcastManager { class PodcastManager {
constructor(watcher, notificationManager, taskManager) { constructor(watcher, notificationManager, taskManager) {
this.watcher = watcher this.watcher = watcher
this.notificationManager = notificationManager this.notificationManager = notificationManager
this.taskManager = taskManager this.taskManager = taskManager
this.downloadQueue = [] this.downloadQueue = []
this.currentDownload = null this.currentDownload = null
this.failedCheckMap = {} this.failedCheckMap = {}
this.MaxFailedEpisodeChecks = 24 this.MaxFailedEpisodeChecks = 24
}
getEpisodeDownloadsInQueue(libraryItemId) {
return this.downloadQueue.filter(d => d.libraryItemId === libraryItemId)
}
clearDownloadQueue(libraryItemId = null) {
if (!this.downloadQueue.length) return
if (!libraryItemId) {
Logger.info(`[PodcastManager] Clearing all downloads in queue (${this.downloadQueue.length})`)
this.downloadQueue = []
} else {
var itemDownloads = this.getEpisodeDownloadsInQueue(libraryItemId)
Logger.info(`[PodcastManager] Clearing downloads in queue for item "${libraryItemId}" (${itemDownloads.length})`)
this.downloadQueue = this.downloadQueue.filter(d => d.libraryItemId !== libraryItemId)
}
}
async downloadPodcastEpisodes(libraryItem, episodesToDownload, isAutoDownload) {
let index = Math.max(...libraryItem.media.episodes.filter(ep => ep.index == null || isNaN(ep.index)).map(ep => Number(ep.index))) + 1
for (const ep of episodesToDownload) {
const newPe = new PodcastEpisode()
newPe.setData(ep, index++)
newPe.libraryItemId = libraryItem.id
newPe.podcastId = libraryItem.media.id
const newPeDl = new PodcastEpisodeDownload()
newPeDl.setData(newPe, libraryItem, isAutoDownload, libraryItem.libraryId)
this.startPodcastEpisodeDownload(newPeDl)
}
}
async startPodcastEpisodeDownload(podcastEpisodeDownload) {
SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails())
if (this.currentDownload) {
this.downloadQueue.push(podcastEpisodeDownload)
SocketAuthority.emitter('episode_download_queued', podcastEpisodeDownload.toJSONForClient())
return
} }
const task = new Task() getEpisodeDownloadsInQueue(libraryItemId) {
const taskDescription = `Downloading episode "${podcastEpisodeDownload.podcastEpisode.title}".` return this.downloadQueue.filter(d => d.libraryItemId === libraryItemId)
const taskData = {
libraryId: podcastEpisodeDownload.libraryId,
libraryItemId: podcastEpisodeDownload.libraryItemId,
}
task.setData('download-podcast-episode', 'Downloading Episode', taskDescription, false, taskData)
this.taskManager.addTask(task)
SocketAuthority.emitter('episode_download_started', podcastEpisodeDownload.toJSONForClient())
this.currentDownload = podcastEpisodeDownload
// If this file already exists then append the episode id to the filename
// e.g. "/tagesschau 20 Uhr.mp3" becomes "/tagesschau 20 Uhr (ep_asdfasdf).mp3"
// this handles podcasts where every title is the same (ref https://github.com/advplyr/audiobookshelf/issues/1802)
if (await fs.pathExists(this.currentDownload.targetPath)) {
this.currentDownload.appendEpisodeId = true
} }
// Ignores all added files to this dir clearDownloadQueue(libraryItemId = null) {
this.watcher.addIgnoreDir(this.currentDownload.libraryItem.path) if (!this.downloadQueue.length) return
// Make sure podcast library item folder exists if (!libraryItemId) {
if (!(await fs.pathExists(this.currentDownload.libraryItem.path))) { Logger.info(`[PodcastManager] Clearing all downloads in queue (${this.downloadQueue.length})`)
Logger.warn(`[PodcastManager] Podcast episode download: Podcast folder no longer exists at "${this.currentDownload.libraryItem.path}" - Creating it`) this.downloadQueue = []
await fs.mkdir(this.currentDownload.libraryItem.path) } else {
} var itemDownloads = this.getEpisodeDownloadsInQueue(libraryItemId)
Logger.info(`[PodcastManager] Clearing downloads in queue for item "${libraryItemId}" (${itemDownloads.length})`)
let success = false this.downloadQueue = this.downloadQueue.filter(d => d.libraryItemId !== libraryItemId)
if (this.currentDownload.urlFileExtension === 'mp3') {
// Download episode and tag it
success = await ffmpegHelpers.downloadPodcastEpisode(this.currentDownload).catch((error) => {
Logger.error(`[PodcastManager] Podcast Episode download failed`, error)
return false
})
} else {
// Download episode only
success = await downloadFile(this.currentDownload.url, this.currentDownload.targetPath).then(() => true).catch((error) => {
Logger.error(`[PodcastManager] Podcast Episode download failed`, error)
return false
})
}
if (success) {
success = await this.scanAddPodcastEpisodeAudioFile()
if (!success) {
await fs.remove(this.currentDownload.targetPath)
this.currentDownload.setFinished(false)
task.setFailed('Failed to download episode')
} else {
Logger.info(`[PodcastManager] Successfully downloaded podcast episode "${this.currentDownload.podcastEpisode.title}"`)
this.currentDownload.setFinished(true)
task.setFinished()
}
} else {
task.setFailed('Failed to download episode')
this.currentDownload.setFinished(false)
}
this.taskManager.taskFinished(task)
SocketAuthority.emitter('episode_download_finished', this.currentDownload.toJSONForClient())
SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails())
this.watcher.removeIgnoreDir(this.currentDownload.libraryItem.path)
this.currentDownload = null
if (this.downloadQueue.length) {
this.startPodcastEpisodeDownload(this.downloadQueue.shift())
}
}
async scanAddPodcastEpisodeAudioFile() {
const libraryFile = await this.getLibraryFile(this.currentDownload.targetPath, this.currentDownload.targetRelPath)
const audioFile = await this.probeAudioFile(libraryFile)
if (!audioFile) {
return false
}
const libraryItem = await Database.libraryItemModel.getOldById(this.currentDownload.libraryItem.id)
if (!libraryItem) {
Logger.error(`[PodcastManager] Podcast Episode finished but library item was not found ${this.currentDownload.libraryItem.id}`)
return false
}
const podcastEpisode = this.currentDownload.podcastEpisode
podcastEpisode.audioFile = audioFile
if (audioFile.chapters?.length) {
podcastEpisode.chapters = audioFile.chapters.map(ch => ({ ...ch }))
}
libraryItem.media.addPodcastEpisode(podcastEpisode)
if (libraryItem.isInvalid) {
// First episode added to an empty podcast
libraryItem.isInvalid = false
}
libraryItem.libraryFiles.push(libraryFile)
if (this.currentDownload.isAutoDownload) {
// Check setting maxEpisodesToKeep and remove episode if necessary
if (libraryItem.media.maxEpisodesToKeep && libraryItem.media.episodesWithPubDate.length > libraryItem.media.maxEpisodesToKeep) {
Logger.info(`[PodcastManager] # of episodes (${libraryItem.media.episodesWithPubDate.length}) exceeds max episodes to keep (${libraryItem.media.maxEpisodesToKeep})`)
await this.removeOldestEpisode(libraryItem, podcastEpisode.id)
}
}
libraryItem.updatedAt = Date.now()
await Database.updateLibraryItem(libraryItem)
SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded())
const podcastEpisodeExpanded = podcastEpisode.toJSONExpanded()
podcastEpisodeExpanded.libraryItem = libraryItem.toJSONExpanded()
SocketAuthority.emitter('episode_added', podcastEpisodeExpanded)
if (this.currentDownload.isAutoDownload) { // Notifications only for auto downloaded episodes
this.notificationManager.onPodcastEpisodeDownloaded(libraryItem, podcastEpisode)
}
return true
}
async removeOldestEpisode(libraryItem, episodeIdJustDownloaded) {
var smallestPublishedAt = 0
var oldestEpisode = null
libraryItem.media.episodesWithPubDate.filter(ep => ep.id !== episodeIdJustDownloaded).forEach((ep) => {
if (!smallestPublishedAt || ep.publishedAt < smallestPublishedAt) {
smallestPublishedAt = ep.publishedAt
oldestEpisode = ep
}
})
// TODO: Should we check for open playback sessions for this episode?
// TODO: remove all user progress for this episode
if (oldestEpisode && oldestEpisode.audioFile) {
Logger.info(`[PodcastManager] Deleting oldest episode "${oldestEpisode.title}"`)
const successfullyDeleted = await removeFile(oldestEpisode.audioFile.metadata.path)
if (successfullyDeleted) {
libraryItem.media.removeEpisode(oldestEpisode.id)
libraryItem.removeLibraryFile(oldestEpisode.audioFile.ino)
return true
} else {
Logger.warn(`[PodcastManager] Failed to remove oldest episode "${oldestEpisode.title}"`)
}
}
return false
}
async getLibraryFile(path, relPath) {
var newLibFile = new LibraryFile()
await newLibFile.setDataFromPath(path, relPath)
return newLibFile
}
async probeAudioFile(libraryFile) {
const path = libraryFile.metadata.path
const mediaProbeData = await prober.probe(path)
if (mediaProbeData.error) {
Logger.error(`[PodcastManager] Podcast Episode downloaded but failed to probe "${path}"`, mediaProbeData.error)
return false
}
const newAudioFile = new AudioFile()
newAudioFile.setDataFromProbe(libraryFile, mediaProbeData)
newAudioFile.index = 1
return newAudioFile
}
// Returns false if auto download episodes was disabled (disabled if reaches max failed checks)
async runEpisodeCheck(libraryItem) {
const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0)
const latestEpisodePublishedAt = libraryItem.media.latestEpisodePublished
Logger.info(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" | Last check: ${lastEpisodeCheckDate} | ${latestEpisodePublishedAt ? `Latest episode pubDate: ${new Date(latestEpisodePublishedAt)}` : 'No latest episode'}`)
// Use latest episode pubDate if exists OR fallback to using lastEpisodeCheckDate
// lastEpisodeCheckDate will be the current time when adding a new podcast
const dateToCheckForEpisodesAfter = latestEpisodePublishedAt || lastEpisodeCheckDate
Logger.debug(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" checking for episodes after ${new Date(dateToCheckForEpisodesAfter)}`)
var newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, dateToCheckForEpisodesAfter, libraryItem.media.maxNewEpisodesToDownload)
Logger.debug(`[PodcastManager] runEpisodeCheck: ${newEpisodes ? newEpisodes.length : 'N/A'} episodes found`)
if (!newEpisodes) { // Failed
// Allow up to MaxFailedEpisodeChecks failed attempts before disabling auto download
if (!this.failedCheckMap[libraryItem.id]) this.failedCheckMap[libraryItem.id] = 0
this.failedCheckMap[libraryItem.id]++
if (this.failedCheckMap[libraryItem.id] >= this.MaxFailedEpisodeChecks) {
Logger.error(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}" - disabling auto download`)
libraryItem.media.autoDownloadEpisodes = false
delete this.failedCheckMap[libraryItem.id]
} else {
Logger.warn(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}"`)
}
} else if (newEpisodes.length) {
delete this.failedCheckMap[libraryItem.id]
Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`)
this.downloadPodcastEpisodes(libraryItem, newEpisodes, true)
} else {
delete this.failedCheckMap[libraryItem.id]
Logger.debug(`[PodcastManager] No new episodes for "${libraryItem.media.metadata.title}"`)
}
libraryItem.media.lastEpisodeCheck = Date.now()
libraryItem.updatedAt = Date.now()
await Database.updateLibraryItem(libraryItem)
SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded())
return libraryItem.media.autoDownloadEpisodes
}
async checkPodcastForNewEpisodes(podcastLibraryItem, dateToCheckForEpisodesAfter, maxNewEpisodes = 3) {
if (!podcastLibraryItem.media.metadata.feedUrl) {
Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`)
return false
}
var feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl)
if (!feed || !feed.episodes) {
Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`, feed)
return false
}
// Filter new and not already has
var newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url))
if (maxNewEpisodes > 0) {
newEpisodes = newEpisodes.slice(0, maxNewEpisodes)
}
return newEpisodes
}
async checkAndDownloadNewEpisodes(libraryItem, maxEpisodesToDownload) {
const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0)
Logger.info(`[PodcastManager] checkAndDownloadNewEpisodes for "${libraryItem.media.metadata.title}" - Last episode check: ${lastEpisodeCheckDate}`)
var newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, libraryItem.media.lastEpisodeCheck, maxEpisodesToDownload)
if (newEpisodes.length) {
Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`)
this.downloadPodcastEpisodes(libraryItem, newEpisodes, false)
} else {
Logger.info(`[PodcastManager] No new episodes found for podcast "${libraryItem.media.metadata.title}"`)
}
libraryItem.media.lastEpisodeCheck = Date.now()
libraryItem.updatedAt = Date.now()
await Database.updateLibraryItem(libraryItem)
SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded())
return newEpisodes
}
async findEpisode(rssFeedUrl, searchTitle) {
const feed = await getPodcastFeed(rssFeedUrl).catch(() => {
return null
})
if (!feed || !feed.episodes) {
return null
}
const matches = []
feed.episodes.forEach(ep => {
if (!ep.title) return
const epTitle = ep.title.toLowerCase().trim()
if (epTitle === searchTitle) {
matches.push({
episode: ep,
levenshtein: 0
})
} else {
const levenshtein = levenshteinDistance(searchTitle, epTitle, true)
if (levenshtein <= 6 && epTitle.length > levenshtein) {
matches.push({
episode: ep,
levenshtein
})
} }
}
})
return matches.sort((a, b) => a.levenshtein - b.levenshtein)
}
async getOPMLFeeds(opmlText) {
var extractedFeeds = opmlParser.parse(opmlText)
if (!extractedFeeds || !extractedFeeds.length) {
Logger.error('[PodcastManager] getOPMLFeeds: No RSS feeds found in OPML')
return {
error: 'No RSS feeds found in OPML'
}
} }
var rssFeedData = [] async downloadPodcastEpisodes(libraryItem, episodesToDownload, isAutoDownload) {
let index = Math.max(...libraryItem.media.episodes.filter(ep => ep.index == null || isNaN(ep.index)).map(ep => Number(ep.index))) + 1
for (let feed of extractedFeeds) { for (const ep of episodesToDownload) {
var feedData = await getPodcastFeed(feed.feedUrl, true) const newPe = new PodcastEpisode()
if (feedData) { newPe.setData(ep, index++)
feedData.metadata.feedUrl = feed.feedUrl newPe.libraryItemId = libraryItem.id
rssFeedData.push(feedData) newPe.podcastId = libraryItem.media.id
} const newPeDl = new PodcastEpisodeDownload()
newPeDl.setData(newPe, libraryItem, isAutoDownload, libraryItem.libraryId)
this.startPodcastEpisodeDownload(newPeDl)
}
} }
return { async startPodcastEpisodeDownload(podcastEpisodeDownload) {
feeds: rssFeedData SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails())
if (this.currentDownload) {
this.downloadQueue.push(podcastEpisodeDownload)
SocketAuthority.emitter('episode_download_queued', podcastEpisodeDownload.toJSONForClient())
return
}
const task = new Task()
const taskDescription = `Downloading episode "${podcastEpisodeDownload.podcastEpisode.title}".`
const taskData = {
libraryId: podcastEpisodeDownload.libraryId,
libraryItemId: podcastEpisodeDownload.libraryItemId,
}
task.setData('download-podcast-episode', 'Downloading Episode', taskDescription, false, taskData)
this.taskManager.addTask(task)
SocketAuthority.emitter('episode_download_started', podcastEpisodeDownload.toJSONForClient())
this.currentDownload = podcastEpisodeDownload
// If this file already exists then append the episode id to the filename
// e.g. "/tagesschau 20 Uhr.mp3" becomes "/tagesschau 20 Uhr (ep_asdfasdf).mp3"
// this handles podcasts where every title is the same (ref https://github.com/advplyr/audiobookshelf/issues/1802)
if (await fs.pathExists(this.currentDownload.targetPath)) {
this.currentDownload.appendEpisodeId = true
}
// Ignores all added files to this dir
this.watcher.addIgnoreDir(this.currentDownload.libraryItem.path)
// Make sure podcast library item folder exists
if (!(await fs.pathExists(this.currentDownload.libraryItem.path))) {
Logger.warn(`[PodcastManager] Podcast episode download: Podcast folder no longer exists at "${this.currentDownload.libraryItem.path}" - Creating it`)
await fs.mkdir(this.currentDownload.libraryItem.path)
}
let success = false
if (this.currentDownload.urlFileExtension === 'mp3') {
// Download episode and tag it
success = await ffmpegHelpers.downloadPodcastEpisode(this.currentDownload).catch((error) => {
Logger.error(`[PodcastManager] Podcast Episode download failed`, error)
return false
})
} else {
// Download episode only
success = await downloadFile(this.currentDownload.url, this.currentDownload.targetPath).then(() => true).catch((error) => {
Logger.error(`[PodcastManager] Podcast Episode download failed`, error)
return false
})
}
if (success) {
success = await this.scanAddPodcastEpisodeAudioFile()
if (!success) {
await fs.remove(this.currentDownload.targetPath)
this.currentDownload.setFinished(false)
task.setFailed('Failed to download episode')
} else {
Logger.info(`[PodcastManager] Successfully downloaded podcast episode "${this.currentDownload.podcastEpisode.title}"`)
this.currentDownload.setFinished(true)
task.setFinished()
}
} else {
task.setFailed('Failed to download episode')
this.currentDownload.setFinished(false)
}
this.taskManager.taskFinished(task)
SocketAuthority.emitter('episode_download_finished', this.currentDownload.toJSONForClient())
SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails())
this.watcher.removeIgnoreDir(this.currentDownload.libraryItem.path)
this.currentDownload = null
if (this.downloadQueue.length) {
this.startPodcastEpisodeDownload(this.downloadQueue.shift())
}
} }
}
/** async scanAddPodcastEpisodeAudioFile() {
* OPML file string for podcasts in a library const libraryFile = await this.getLibraryFile(this.currentDownload.targetPath, this.currentDownload.targetRelPath)
* @param {import('../models/Podcast')[]} podcasts
* @returns {string} XML string
*/
generateOPMLFileText(podcasts) {
return opmlGenerator.generate(podcasts)
}
getDownloadQueueDetails(libraryId = null) { const audioFile = await this.probeAudioFile(libraryFile)
let _currentDownload = this.currentDownload if (!audioFile) {
if (libraryId && _currentDownload?.libraryId !== libraryId) _currentDownload = null return false
}
return { const libraryItem = await Database.libraryItemModel.getOldById(this.currentDownload.libraryItem.id)
currentDownload: _currentDownload?.toJSONForClient(), if (!libraryItem) {
queue: this.downloadQueue.filter(item => !libraryId || item.libraryId === libraryId).map(item => item.toJSONForClient()) Logger.error(`[PodcastManager] Podcast Episode finished but library item was not found ${this.currentDownload.libraryItem.id}`)
return false
}
const podcastEpisode = this.currentDownload.podcastEpisode
podcastEpisode.audioFile = audioFile
if (audioFile.chapters?.length) {
podcastEpisode.chapters = audioFile.chapters.map(ch => ({...ch}))
}
libraryItem.media.addPodcastEpisode(podcastEpisode)
if (libraryItem.isInvalid) {
// First episode added to an empty podcast
libraryItem.isInvalid = false
}
libraryItem.libraryFiles.push(libraryFile)
if (this.currentDownload.isAutoDownload) {
// Check setting maxEpisodesToKeep and remove episode if necessary
if (libraryItem.media.maxEpisodesToKeep && libraryItem.media.episodesWithPubDate.length > libraryItem.media.maxEpisodesToKeep) {
Logger.info(`[PodcastManager] # of episodes (${libraryItem.media.episodesWithPubDate.length}) exceeds max episodes to keep (${libraryItem.media.maxEpisodesToKeep})`)
await this.removeOldestEpisode(libraryItem, podcastEpisode.id)
}
}
libraryItem.updatedAt = Date.now()
await Database.updateLibraryItem(libraryItem)
SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded())
const podcastEpisodeExpanded = podcastEpisode.toJSONExpanded()
podcastEpisodeExpanded.libraryItem = libraryItem.toJSONExpanded()
SocketAuthority.emitter('episode_added', podcastEpisodeExpanded)
if (this.currentDownload.isAutoDownload) { // Notifications only for auto downloaded episodes
this.notificationManager.onPodcastEpisodeDownloaded(libraryItem, podcastEpisode)
}
return true
}
async removeOldestEpisode(libraryItem, episodeIdJustDownloaded) {
var smallestPublishedAt = 0
var oldestEpisode = null
libraryItem.media.episodesWithPubDate.filter(ep => ep.id !== episodeIdJustDownloaded).forEach((ep) => {
if (!smallestPublishedAt || ep.publishedAt < smallestPublishedAt) {
smallestPublishedAt = ep.publishedAt
oldestEpisode = ep
}
})
// TODO: Should we check for open playback sessions for this episode?
// TODO: remove all user progress for this episode
if (oldestEpisode && oldestEpisode.audioFile) {
Logger.info(`[PodcastManager] Deleting oldest episode "${oldestEpisode.title}"`)
const successfullyDeleted = await removeFile(oldestEpisode.audioFile.metadata.path)
if (successfullyDeleted) {
libraryItem.media.removeEpisode(oldestEpisode.id)
libraryItem.removeLibraryFile(oldestEpisode.audioFile.ino)
return true
} else {
Logger.warn(`[PodcastManager] Failed to remove oldest episode "${oldestEpisode.title}"`)
}
}
return false
}
async getLibraryFile(path, relPath) {
var newLibFile = new LibraryFile()
await newLibFile.setDataFromPath(path, relPath)
return newLibFile
}
async probeAudioFile(libraryFile) {
const path = libraryFile.metadata.path
const mediaProbeData = await prober.probe(path)
if (mediaProbeData.error) {
Logger.error(`[PodcastManager] Podcast Episode downloaded but failed to probe "${path}"`, mediaProbeData.error)
return false
}
const newAudioFile = new AudioFile()
newAudioFile.setDataFromProbe(libraryFile, mediaProbeData)
newAudioFile.index = 1
return newAudioFile
}
// Returns false if auto download episodes was disabled (disabled if reaches max failed checks)
async runEpisodeCheck(libraryItem) {
const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0)
const latestEpisodePublishedAt = libraryItem.media.latestEpisodePublished
Logger.info(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" | Last check: ${lastEpisodeCheckDate} | ${latestEpisodePublishedAt ? `Latest episode pubDate: ${new Date(latestEpisodePublishedAt)}` : 'No latest episode'}`)
// Use latest episode pubDate if exists OR fallback to using lastEpisodeCheckDate
// lastEpisodeCheckDate will be the current time when adding a new podcast
const dateToCheckForEpisodesAfter = latestEpisodePublishedAt || lastEpisodeCheckDate
Logger.debug(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" checking for episodes after ${new Date(dateToCheckForEpisodesAfter)}`)
let newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, dateToCheckForEpisodesAfter, libraryItem.media.maxNewEpisodesToDownload)
Logger.debug(`[PodcastManager] runEpisodeCheck: ${newEpisodes ? newEpisodes.length : 'N/A'} episodes found`)
if (!newEpisodes) { // Failed
// Allow up to MaxFailedEpisodeChecks failed attempts before disabling auto download
if (!this.failedCheckMap[libraryItem.id]) this.failedCheckMap[libraryItem.id] = 0
this.failedCheckMap[libraryItem.id]++
if (this.failedCheckMap[libraryItem.id] >= this.MaxFailedEpisodeChecks) {
Logger.error(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}" - disabling auto download`)
libraryItem.media.autoDownloadEpisodes = false
delete this.failedCheckMap[libraryItem.id]
} else {
Logger.warn(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}"`)
}
libraryItem.media.metadata.feedHealthy = false
} else if (newEpisodes.length) {
delete this.failedCheckMap[libraryItem.id]
Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`)
this.downloadPodcastEpisodes(libraryItem, newEpisodes, true)
libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now()
libraryItem.media.metadata.feedHealthy = true
} else {
delete this.failedCheckMap[libraryItem.id]
Logger.debug(`[PodcastManager] No new episodes for "${libraryItem.media.metadata.title}"`)
libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now()
libraryItem.media.metadata.feedHealthy = true
}
libraryItem.media.lastEpisodeCheck = Date.now()
libraryItem.updatedAt = Date.now()
await Database.updateLibraryItem(libraryItem)
SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded())
return libraryItem.media.autoDownloadEpisodes
}
async checkPodcastForNewEpisodes(podcastLibraryItem, dateToCheckForEpisodesAfter, maxNewEpisodes = 3) {
if (!podcastLibraryItem.media.metadata.feedUrl) {
Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`)
return false
}
var feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl)
if (!feed || !feed.episodes) {
Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id}, URL: ${podcastLibraryItem.media.metadata.feedUrl})`, feed)
return false
}
// Filter new and not already has
var newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url))
if (maxNewEpisodes > 0) {
newEpisodes = newEpisodes.slice(0, maxNewEpisodes)
}
return newEpisodes
}
async checkAndDownloadNewEpisodes(libraryItem, maxEpisodesToDownload) {
const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0)
Logger.info(`[PodcastManager] checkAndDownloadNewEpisodes for "${libraryItem.media.metadata.title}" - Last episode check: ${lastEpisodeCheckDate}`)
let newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, libraryItem.media.lastEpisodeCheck, maxEpisodesToDownload)
if (!newEpisodes) {
libraryItem.media.metadata.feedHealthy = false
} else if (newEpisodes.length) {
Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`)
this.downloadPodcastEpisodes(libraryItem, newEpisodes, false)
libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now()
libraryItem.media.metadata.feedHealthy = true
} else {
Logger.info(`[PodcastManager] No new episodes found for podcast "${libraryItem.media.metadata.title}"`)
libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now()
libraryItem.media.metadata.feedHealthy = true
}
libraryItem.media.lastEpisodeCheck = Date.now()
libraryItem.updatedAt = Date.now()
await Database.updateLibraryItem(libraryItem)
SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded())
return newEpisodes
}
setFeedHealthStatus(libraryItem, isHealthy) {
libraryItem.media.metadata.feedHealthy = isHealthy
if (isHealthy) {
libraryItem.media.lastSuccessfulFetchAt = Date.now()
}
libraryItem.updatedAt = Date.now()
Database.updateLibraryItem(libraryItem)
}
async findEpisode(rssFeedUrl, searchTitle) {
const feed = await getPodcastFeed(rssFeedUrl).catch(() => {
return null
})
if (!feed || !feed.episodes) {
return null
}
const matches = []
feed.episodes.forEach(ep => {
if (!ep.title) return
const epTitle = ep.title.toLowerCase().trim()
if (epTitle === searchTitle) {
matches.push({
episode: ep,
levenshtein: 0
})
} else {
const levenshtein = levenshteinDistance(searchTitle, epTitle, true)
if (levenshtein <= 6 && epTitle.length > levenshtein) {
matches.push({
episode: ep,
levenshtein
})
}
}
})
return matches.sort((a, b) => a.levenshtein - b.levenshtein)
}
async getOPMLFeeds(opmlText) {
var extractedFeeds = opmlParser.parse(opmlText)
if (!extractedFeeds || !extractedFeeds.length) {
Logger.error('[PodcastManager] getOPMLFeeds: No RSS feeds found in OPML')
return {
error: 'No RSS feeds found in OPML'
}
}
var rssFeedData = []
for (let feed of extractedFeeds) {
var feedData = await getPodcastFeed(feed.feedUrl, true)
if (feedData) {
feedData.metadata.feedUrl = feed.feedUrl
rssFeedData.push(feedData)
}
}
return {
feeds: rssFeedData
}
}
/**
* OPML file string for podcasts in a library
* @param {import('../models/Podcast')[]} podcasts
* @returns {string} XML string
*/
generateOPMLFileText(podcasts) {
return opmlGenerator.generate(podcasts)
}
getDownloadQueueDetails(libraryId = null) {
let _currentDownload = this.currentDownload
if (libraryId && _currentDownload?.libraryId !== libraryId) _currentDownload = null
return {
currentDownload: _currentDownload?.toJSONForClient(),
queue: this.downloadQueue.filter(item => !libraryId || item.libraryId === libraryId).map(item => item.toJSONForClient())
}
} }
}
} }
module.exports = PodcastManager module.exports = PodcastManager

View file

@ -50,6 +50,10 @@ class Podcast extends Model {
this.createdAt this.createdAt
/** @type {Date} */ /** @type {Date} */
this.updatedAt this.updatedAt
/** @type {Date} */
this.lastSuccessfulFetchAt
/** @type {boolean} */
this.feedHealthy
} }
static getOldPodcast(libraryItemExpanded) { static getOldPodcast(libraryItemExpanded) {
@ -71,7 +75,9 @@ class Podcast extends Model {
itunesArtistId: podcastExpanded.itunesArtistId, itunesArtistId: podcastExpanded.itunesArtistId,
explicit: podcastExpanded.explicit, explicit: podcastExpanded.explicit,
language: podcastExpanded.language, language: podcastExpanded.language,
type: podcastExpanded.podcastType type: podcastExpanded.podcastType,
lastSuccessfulFetchAt: podcastExpanded.lastSuccessfulFetchAt?.valueOf() || null,
feedHealthy: !!podcastExpanded.feedHealthy
}, },
coverPath: podcastExpanded.coverPath, coverPath: podcastExpanded.coverPath,
tags: podcastExpanded.tags, tags: podcastExpanded.tags,
@ -108,7 +114,9 @@ class Podcast extends Model {
maxNewEpisodesToDownload: oldPodcast.maxNewEpisodesToDownload, maxNewEpisodesToDownload: oldPodcast.maxNewEpisodesToDownload,
coverPath: oldPodcast.coverPath, coverPath: oldPodcast.coverPath,
tags: oldPodcast.tags, tags: oldPodcast.tags,
genres: oldPodcastMetadata.genres genres: oldPodcastMetadata.genres,
lastSuccessfulFetchAt: oldPodcastMetadata.lastSuccessfulFetchAt,
feedHealthy: !!oldPodcastMetadata.feedHealthy
} }
} }
@ -144,7 +152,9 @@ class Podcast extends Model {
maxNewEpisodesToDownload: DataTypes.INTEGER, maxNewEpisodesToDownload: DataTypes.INTEGER,
coverPath: DataTypes.STRING, coverPath: DataTypes.STRING,
tags: DataTypes.JSON, tags: DataTypes.JSON,
genres: DataTypes.JSON genres: DataTypes.JSON,
lastSuccessfulFetchAt: DataTypes.DATE,
feedHealthy: DataTypes.BOOLEAN
}, { }, {
sequelize, sequelize,
modelName: 'podcast' modelName: 'podcast'
@ -152,4 +162,4 @@ class Podcast extends Model {
} }
} }
module.exports = Podcast module.exports = Podcast

View file

@ -226,6 +226,7 @@ class ApiRouter {
// //
this.router.post('/podcasts', PodcastController.create.bind(this)) this.router.post('/podcasts', PodcastController.create.bind(this))
this.router.post('/podcasts/feed', PodcastController.getPodcastFeed.bind(this)) this.router.post('/podcasts/feed', PodcastController.getPodcastFeed.bind(this))
this.router.get('/podcasts/:id/feed', PodcastController.middleware.bind(this), PodcastController.checkPodcastFeed.bind(this))
this.router.post('/podcasts/opml', PodcastController.getFeedsFromOPMLText.bind(this)) this.router.post('/podcasts/opml', PodcastController.getFeedsFromOPMLText.bind(this))
this.router.get('/podcasts/:id/checknew', PodcastController.middleware.bind(this), PodcastController.checkNewEpisodes.bind(this)) this.router.get('/podcasts/:id/checknew', PodcastController.middleware.bind(this), PodcastController.checkNewEpisodes.bind(this))
this.router.get('/podcasts/:id/downloads', PodcastController.middleware.bind(this), PodcastController.getEpisodeDownloads.bind(this)) this.router.get('/podcasts/:id/downloads', PodcastController.middleware.bind(this), PodcastController.getEpisodeDownloads.bind(this))

View file

@ -198,7 +198,9 @@ function migratePodcast(oldLibraryItem, LibraryItem) {
updatedAt: LibraryItem.updatedAt, updatedAt: LibraryItem.updatedAt,
coverPath: oldPodcast.coverPath, coverPath: oldPodcast.coverPath,
tags: oldPodcast.tags, tags: oldPodcast.tags,
genres: oldPodcastMetadata.genres genres: oldPodcastMetadata.genres,
lastSuccessfulFetchAt: oldPodcastMetadata.lastSuccessfulFetchAt || null,
feedHealthy: !!oldPodcastMetadata.feedHealthy || null
} }
_newRecords.podcast = Podcast _newRecords.podcast = Podcast
oldDbIdMap.podcasts[oldLibraryItem.id] = Podcast.id oldDbIdMap.podcasts[oldLibraryItem.id] = Podcast.id
@ -1708,4 +1710,4 @@ module.exports.migrationPatch2 = async (ctx) => {
Logger.error(`[dbMigration] Migration from 2.3.3+ column creation failed`, error) Logger.error(`[dbMigration] Migration from 2.3.3+ column creation failed`, error)
throw new Error('Migration 2.3.3+ failed ' + error) throw new Error('Migration 2.3.3+ failed ' + error)
} }
} }