mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2026-05-23 20:01:35 +00:00
- Add new migration to add an autoGenerateChapters column in the Podcasts table
- Bump minor version (I wasn't sure if this was needed for the migration) - Feature is now controlled by the field in the podcast database object - Move parsing code and tests to existing utils/parsers/ dir - Add more test cases
This commit is contained in:
parent
12b04faed2
commit
4907e70a48
10 changed files with 393 additions and 200 deletions
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "audiobookshelf-client",
|
"name": "audiobookshelf-client",
|
||||||
"version": "2.33.0",
|
"version": "2.34.0",
|
||||||
"buildNumber": 1,
|
"buildNumber": 1,
|
||||||
"description": "Self-hosted audiobook and podcast client",
|
"description": "Self-hosted audiobook and podcast client",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "audiobookshelf",
|
"name": "audiobookshelf",
|
||||||
"version": "2.33.0",
|
"version": "2.34.0",
|
||||||
"buildNumber": 1,
|
"buildNumber": 1,
|
||||||
"description": "Self-hosted audiobook and podcast server",
|
"description": "Self-hosted audiobook and podcast server",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
|
|
|
||||||
|
|
@ -204,7 +204,7 @@ class PodcastManager {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
const podcastEpisode = await Database.podcastEpisodeModel.createFromRssPodcastEpisode(this.currentDownload.rssPodcastEpisode, libraryItem.media.id, audioFile)
|
const podcastEpisode = await Database.podcastEpisodeModel.createFromRssPodcastEpisode(this.currentDownload.rssPodcastEpisode, libraryItem.media.id, libraryItem.media.autoGenerateChapters, audioFile)
|
||||||
|
|
||||||
libraryItem.libraryFiles.push(libraryFile.toJSON())
|
libraryItem.libraryFiles.push(libraryFile.toJSON())
|
||||||
// Re-calculating library item size because this wasnt being updated properly for podcasts in v2.20.0 and below
|
// Re-calculating library item size because this wasnt being updated properly for podcasts in v2.20.0 and below
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,83 @@
|
||||||
|
const util = require('util')
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef MigrationContext
|
||||||
|
* @property {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
|
||||||
|
* @property {import('../Logger')} logger - a Logger object.
|
||||||
|
*
|
||||||
|
* @typedef MigrationOptions
|
||||||
|
* @property {MigrationContext} context - an object containing the migration context.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const migrationVersion = '2.34.0'
|
||||||
|
const migrationName = `${migrationVersion}-add-auto-generate-podcast-chapters`
|
||||||
|
const loggerPrefix = `[${migrationVersion} migration]`
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This upward migration adds a boolean autoGenerateChapters column to the podcasts table and defaults it to false.
|
||||||
|
*
|
||||||
|
* @param {MigrationOptions} options - an object containing the migration context.
|
||||||
|
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
|
||||||
|
*/
|
||||||
|
async function up({ context: { queryInterface, logger } }) {
|
||||||
|
logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
|
||||||
|
|
||||||
|
await addColumn(queryInterface, logger, 'podcasts', 'autoGenerateChapters', { type: queryInterface.sequelize.Sequelize.BOOLEAN, allowNull: false, defaultValue: false })
|
||||||
|
|
||||||
|
logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This downward migration removes the autoGenerateChapters column on the podcasts table,
|
||||||
|
*
|
||||||
|
* @param {MigrationOptions} options - an object containing the migration context.
|
||||||
|
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
|
||||||
|
*/
|
||||||
|
async function down({ context: { queryInterface, logger } }) {
|
||||||
|
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
|
||||||
|
|
||||||
|
await removeColumn(queryInterface, logger, 'podcasts', 'autoGenerateChapters')
|
||||||
|
|
||||||
|
logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Utility function to add a column to a table. If the column already exists, it logs a message and continues.
|
||||||
|
*
|
||||||
|
* @param {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
|
||||||
|
* @param {import('../Logger')} logger - a Logger object.
|
||||||
|
* @param {string} table - the name of the table to add the column to.
|
||||||
|
* @param {string} column - the name of the column to add.
|
||||||
|
* @param {Object} options - the options for the column.
|
||||||
|
*/
|
||||||
|
async function addColumn(queryInterface, logger, table, column, options) {
|
||||||
|
logger.info(`${loggerPrefix} adding column "${column}" to table "${table}"`)
|
||||||
|
const tableDescription = await queryInterface.describeTable(table)
|
||||||
|
if (!tableDescription[column]) {
|
||||||
|
await queryInterface.addColumn(table, column, options)
|
||||||
|
logger.info(`${loggerPrefix} added column "${column}" to table "${table}"`)
|
||||||
|
} else {
|
||||||
|
logger.info(`${loggerPrefix} column "${column}" already exists in table "${table}"`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Utility function to remove a column from a table. If the column does not exist, it logs a message and continues.
|
||||||
|
*
|
||||||
|
* @param {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
|
||||||
|
* @param {import('../Logger')} logger - a Logger object.
|
||||||
|
* @param {string} table - the name of the table to remove the column from.
|
||||||
|
* @param {string} column - the name of the column to remove.
|
||||||
|
*/
|
||||||
|
async function removeColumn(queryInterface, logger, table, column) {
|
||||||
|
logger.info(`${loggerPrefix} removing column "${column}" from table "${table}"`)
|
||||||
|
const tableDescription = await queryInterface.describeTable(table)
|
||||||
|
if (tableDescription[column]) {
|
||||||
|
await queryInterface.sequelize.query(`ALTER TABLE ${table} DROP COLUMN ${column}`)
|
||||||
|
logger.info(`${loggerPrefix} removed column "${column}" from table "${table}"`)
|
||||||
|
} else {
|
||||||
|
logger.info(`${loggerPrefix} column "${column}" does not exist in table "${table}"`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { up, down }
|
||||||
|
|
@ -53,6 +53,8 @@ class Podcast extends Model {
|
||||||
this.maxEpisodesToKeep
|
this.maxEpisodesToKeep
|
||||||
/** @type {number} */
|
/** @type {number} */
|
||||||
this.maxNewEpisodesToDownload
|
this.maxNewEpisodesToDownload
|
||||||
|
/** @type {boolean} */
|
||||||
|
this.autoGenerateChapters
|
||||||
/** @type {string} */
|
/** @type {string} */
|
||||||
this.coverPath
|
this.coverPath
|
||||||
/** @type {string[]} */
|
/** @type {string[]} */
|
||||||
|
|
@ -106,6 +108,7 @@ class Podcast extends Model {
|
||||||
explicit: !!payload.metadata.explicit,
|
explicit: !!payload.metadata.explicit,
|
||||||
autoDownloadEpisodes: !!payload.autoDownloadEpisodes,
|
autoDownloadEpisodes: !!payload.autoDownloadEpisodes,
|
||||||
autoDownloadSchedule: autoDownloadSchedule || global.ServerSettings.podcastEpisodeSchedule,
|
autoDownloadSchedule: autoDownloadSchedule || global.ServerSettings.podcastEpisodeSchedule,
|
||||||
|
autoGenerateChapters: !!payload.autoGenerateChapters,
|
||||||
lastEpisodeCheck: new Date(),
|
lastEpisodeCheck: new Date(),
|
||||||
maxEpisodesToKeep: 0,
|
maxEpisodesToKeep: 0,
|
||||||
maxNewEpisodesToDownload: 3,
|
maxNewEpisodesToDownload: 3,
|
||||||
|
|
@ -145,6 +148,7 @@ class Podcast extends Model {
|
||||||
autoDownloadEpisodes: DataTypes.BOOLEAN,
|
autoDownloadEpisodes: DataTypes.BOOLEAN,
|
||||||
autoDownloadSchedule: DataTypes.STRING,
|
autoDownloadSchedule: DataTypes.STRING,
|
||||||
lastEpisodeCheck: DataTypes.DATE,
|
lastEpisodeCheck: DataTypes.DATE,
|
||||||
|
autoGenerateChapters: DataTypes.BOOLEAN,
|
||||||
maxEpisodesToKeep: DataTypes.INTEGER,
|
maxEpisodesToKeep: DataTypes.INTEGER,
|
||||||
maxNewEpisodesToDownload: DataTypes.INTEGER,
|
maxNewEpisodesToDownload: DataTypes.INTEGER,
|
||||||
coverPath: DataTypes.STRING,
|
coverPath: DataTypes.STRING,
|
||||||
|
|
@ -273,6 +277,10 @@ class Podcast extends Model {
|
||||||
this.autoDownloadSchedule = payload.autoDownloadSchedule
|
this.autoDownloadSchedule = payload.autoDownloadSchedule
|
||||||
hasUpdates = true
|
hasUpdates = true
|
||||||
}
|
}
|
||||||
|
if (payload.autoGenerateChapters !== undefined && payload.autoGenerateChapters !== this.autoGenerateChapters) {
|
||||||
|
this.autoGenerateChapters = !!payload.autoGenerateChapters
|
||||||
|
hasUpdates = true
|
||||||
|
}
|
||||||
if (typeof payload.lastEpisodeCheck === 'number' && payload.lastEpisodeCheck !== this.lastEpisodeCheck?.valueOf()) {
|
if (typeof payload.lastEpisodeCheck === 'number' && payload.lastEpisodeCheck !== this.lastEpisodeCheck?.valueOf()) {
|
||||||
this.lastEpisodeCheck = payload.lastEpisodeCheck
|
this.lastEpisodeCheck = payload.lastEpisodeCheck
|
||||||
hasUpdates = true
|
hasUpdates = true
|
||||||
|
|
@ -441,6 +449,7 @@ class Podcast extends Model {
|
||||||
autoDownloadEpisodes: this.autoDownloadEpisodes,
|
autoDownloadEpisodes: this.autoDownloadEpisodes,
|
||||||
autoDownloadSchedule: this.autoDownloadSchedule,
|
autoDownloadSchedule: this.autoDownloadSchedule,
|
||||||
lastEpisodeCheck: this.lastEpisodeCheck?.valueOf() || null,
|
lastEpisodeCheck: this.lastEpisodeCheck?.valueOf() || null,
|
||||||
|
autoGenerateChapters: this.autoGenerateChapters,
|
||||||
maxEpisodesToKeep: this.maxEpisodesToKeep,
|
maxEpisodesToKeep: this.maxEpisodesToKeep,
|
||||||
maxNewEpisodesToDownload: this.maxNewEpisodesToDownload
|
maxNewEpisodesToDownload: this.maxNewEpisodesToDownload
|
||||||
}
|
}
|
||||||
|
|
@ -457,6 +466,7 @@ class Podcast extends Model {
|
||||||
autoDownloadEpisodes: this.autoDownloadEpisodes,
|
autoDownloadEpisodes: this.autoDownloadEpisodes,
|
||||||
autoDownloadSchedule: this.autoDownloadSchedule,
|
autoDownloadSchedule: this.autoDownloadSchedule,
|
||||||
lastEpisodeCheck: this.lastEpisodeCheck?.valueOf() || null,
|
lastEpisodeCheck: this.lastEpisodeCheck?.valueOf() || null,
|
||||||
|
autoGenerateChapters: this.autoGenerateChapters,
|
||||||
maxEpisodesToKeep: this.maxEpisodesToKeep,
|
maxEpisodesToKeep: this.maxEpisodesToKeep,
|
||||||
maxNewEpisodesToDownload: this.maxNewEpisodesToDownload,
|
maxNewEpisodesToDownload: this.maxNewEpisodesToDownload,
|
||||||
size: this.size
|
size: this.size
|
||||||
|
|
@ -481,6 +491,7 @@ class Podcast extends Model {
|
||||||
autoDownloadEpisodes: this.autoDownloadEpisodes,
|
autoDownloadEpisodes: this.autoDownloadEpisodes,
|
||||||
autoDownloadSchedule: this.autoDownloadSchedule,
|
autoDownloadSchedule: this.autoDownloadSchedule,
|
||||||
lastEpisodeCheck: this.lastEpisodeCheck?.valueOf() || null,
|
lastEpisodeCheck: this.lastEpisodeCheck?.valueOf() || null,
|
||||||
|
autoGenerateChapters: this.autoGenerateChapters,
|
||||||
maxEpisodesToKeep: this.maxEpisodesToKeep,
|
maxEpisodesToKeep: this.maxEpisodesToKeep,
|
||||||
maxNewEpisodesToDownload: this.maxNewEpisodesToDownload,
|
maxNewEpisodesToDownload: this.maxNewEpisodesToDownload,
|
||||||
size: this.size
|
size: this.size
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
const { DataTypes, Model } = require('sequelize')
|
const { DataTypes, Model } = require('sequelize')
|
||||||
const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters')
|
const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters')
|
||||||
|
const parsePodcastDescriptionForChapters = require('../utils/parsers/parsePodcastDescriptionForChapters')
|
||||||
const Logger = require('../Logger')
|
const Logger = require('../Logger')
|
||||||
/**
|
/**
|
||||||
* @typedef ChapterObject
|
* @typedef ChapterObject
|
||||||
|
|
@ -57,9 +58,10 @@ class PodcastEpisode extends Model {
|
||||||
*
|
*
|
||||||
* @param {import('../utils/podcastUtils').RssPodcastEpisode} rssPodcastEpisode
|
* @param {import('../utils/podcastUtils').RssPodcastEpisode} rssPodcastEpisode
|
||||||
* @param {string} podcastId
|
* @param {string} podcastId
|
||||||
|
* @param {boolean} autoGenerateChapters
|
||||||
* @param {import('../objects/files/AudioFile')} audioFile
|
* @param {import('../objects/files/AudioFile')} audioFile
|
||||||
*/
|
*/
|
||||||
static async createFromRssPodcastEpisode(rssPodcastEpisode, podcastId, audioFile) {
|
static async createFromRssPodcastEpisode(rssPodcastEpisode, podcastId, autoGenerateChapters, audioFile) {
|
||||||
const podcastEpisode = {
|
const podcastEpisode = {
|
||||||
index: null,
|
index: null,
|
||||||
season: rssPodcastEpisode.season,
|
season: rssPodcastEpisode.season,
|
||||||
|
|
@ -86,11 +88,10 @@ class PodcastEpisode extends Model {
|
||||||
podcastEpisode.chapters = audioFile.chapters.map((ch) => ({ ...ch }))
|
podcastEpisode.chapters = audioFile.chapters.map((ch) => ({ ...ch }))
|
||||||
} else if (rssPodcastEpisode.chapters?.length) {
|
} else if (rssPodcastEpisode.chapters?.length) {
|
||||||
podcastEpisode.chapters = rssPodcastEpisode.chapters.map((ch) => ({ ...ch }))
|
podcastEpisode.chapters = rssPodcastEpisode.chapters.map((ch) => ({ ...ch }))
|
||||||
} else {
|
} else if (autoGenerateChapters) {
|
||||||
Logger.info("[PodcastEpisode] New episode doesn't have chapters, attempting to generate them from timestamps", rssPodcastEpisode.title)
|
Logger.info("[PodcastEpisode] New episode doesn't have chapters, attempting to generate them from timestamps", rssPodcastEpisode.title)
|
||||||
try {
|
try {
|
||||||
let autoGeneratedChapters = PodcastEpisode.autoGenerateChaptersFromTimestamps(podcastEpisode.description, podcastEpisode.audioFile.duration)
|
podcastEpisode.chapters = parsePodcastDescriptionForChapters.parse(podcastEpisode.description, podcastEpisode.audioFile.duration)
|
||||||
podcastEpisode.chapters = autoGeneratedChapters
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
Logger.error(`[PodcastEpisode] createFromRssPodcastEpisode: Failed to auto generate chapters for "${podcastEpisode.title}"`, error)
|
Logger.error(`[PodcastEpisode] createFromRssPodcastEpisode: Failed to auto generate chapters for "${podcastEpisode.title}"`, error)
|
||||||
}
|
}
|
||||||
|
|
@ -245,82 +246,6 @@ class PodcastEpisode extends Model {
|
||||||
|
|
||||||
return json
|
return json
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* @param {string} podcastDescription
|
|
||||||
* @param {number} audioDurationSecs
|
|
||||||
* @returns {ChapterObject[]}
|
|
||||||
*/
|
|
||||||
static autoGenerateChaptersFromTimestamps(podcastDescription, audioDurationSecs) {
|
|
||||||
if (podcastDescription == null) {
|
|
||||||
throw new Error('Description must not be null')
|
|
||||||
}
|
|
||||||
|
|
||||||
if (audioDurationSecs == null) {
|
|
||||||
throw new Error('Audio duration must not be null')
|
|
||||||
}
|
|
||||||
|
|
||||||
const timestampRegex = /\b(\d{1,2}):(\d{1,2})(?::(\d{1,2}))?\b/
|
|
||||||
const chapterTitleRegex = /\b\d{1,2}:\d{1,2}(?::\d{1,2})?\b(?:\s+|\))(.+)$/
|
|
||||||
const descriptionLineSplitRegex = /\<\s*\/\s*p\s*\>|\<\s*br\s*\s*\/\>|\n/
|
|
||||||
|
|
||||||
var descriptionLines = podcastDescription.split(descriptionLineSplitRegex)
|
|
||||||
var newChapters = []
|
|
||||||
|
|
||||||
for (let i = 0; i < descriptionLines.length; i++) {
|
|
||||||
let line = descriptionLines[i]
|
|
||||||
|
|
||||||
let match = timestampRegex.exec(line)
|
|
||||||
if (match == null) continue
|
|
||||||
|
|
||||||
let first = match[1]
|
|
||||||
let second = match[2]
|
|
||||||
let third = match[3]
|
|
||||||
|
|
||||||
let hours = 0
|
|
||||||
let minutes = 0
|
|
||||||
let seconds = 0
|
|
||||||
|
|
||||||
// If there's three components then we can assume its hh:mm:ss
|
|
||||||
if (first && second && third) {
|
|
||||||
hours = Number(first)
|
|
||||||
minutes = Number(second)
|
|
||||||
seconds = Number(third)
|
|
||||||
} else if (first && second) // otherwise assume mm:ss
|
|
||||||
{
|
|
||||||
minutes = Number(first)
|
|
||||||
seconds = Number(second)
|
|
||||||
}
|
|
||||||
|
|
||||||
let startTime = seconds + minutes * 60 + hours * 60 * 60
|
|
||||||
let chapterTitleMatch = chapterTitleRegex.exec(line)
|
|
||||||
|
|
||||||
if (chapterTitleMatch == null || chapterTitleMatch.length < 2) {
|
|
||||||
// Unknown chapter state
|
|
||||||
throw new Error(`Unable to get chapter title from description, line ${line}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
let chapter = { title: chapterTitleMatch[1].trim(), id: newChapters.length + 1, start: startTime }
|
|
||||||
|
|
||||||
if (newChapters.length > 0) {
|
|
||||||
newChapters[newChapters.length - 1].end = startTime
|
|
||||||
}
|
|
||||||
|
|
||||||
newChapters.push(chapter)
|
|
||||||
}
|
|
||||||
if (newChapters.length > 0) {
|
|
||||||
newChapters[newChapters.length - 1].end = audioDurationSecs
|
|
||||||
}
|
|
||||||
|
|
||||||
Logger.info(`[PodcastEpisode] Successfully generated ${newChapters.length} chapters`)
|
|
||||||
|
|
||||||
if (newChapters.length == 1) {
|
|
||||||
throw new Error('Only one chapter found, treating as invalid description')
|
|
||||||
}
|
|
||||||
|
|
||||||
return newChapters
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = PodcastEpisode
|
module.exports = PodcastEpisode
|
||||||
|
|
|
||||||
|
|
@ -1,117 +0,0 @@
|
||||||
const chai = require('chai')
|
|
||||||
const PodcastEpisode = require('./PodcastEpisode')
|
|
||||||
const Logger = require('../Logger')
|
|
||||||
const expect = chai.expect
|
|
||||||
|
|
||||||
describe('PodcastEpisode', () => {
|
|
||||||
describe('autoGenerateChaptersFromTimestamps', () => {
|
|
||||||
var testCasesTestingSuccess = [
|
|
||||||
{
|
|
||||||
testName: 'Should handle descriptions using html paragraphs',
|
|
||||||
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>00:48 Chatper 1 </p><p>12:14 Chapter 2 </p><p>20:56 Chapter 3 </p><p>27:34 Chapter 4 </p><p>32:00 Chapter 5 </p><p>35:16 Chapter 6 </p><p>41:32 Chapter 7 </p><p>46:43 Chapter 8</p>',
|
|
||||||
audioDuration: 3060,
|
|
||||||
expectedChapters: [
|
|
||||||
{ title: 'Chatper 1', id: 1, start: 48, end: 734 },
|
|
||||||
{ title: 'Chapter 2', id: 2, start: 734, end: 1256 },
|
|
||||||
{ title: 'Chapter 3', id: 3, start: 1256, end: 1654 },
|
|
||||||
{ title: 'Chapter 4', id: 4, start: 1654, end: 1920 },
|
|
||||||
{ title: 'Chapter 5', id: 5, start: 1920, end: 2116 },
|
|
||||||
{ title: 'Chapter 6', id: 6, start: 2116, end: 2492 },
|
|
||||||
{ title: 'Chapter 7', id: 7, start: 2492, end: 2803 },
|
|
||||||
{ title: 'Chapter 8', id: 8, start: 2803, end: 3060 }
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
testName: 'Should handle descriptions using html line breaks',
|
|
||||||
description: '<br>Introduction text paragraph 1<br /><br>Introduction text paragraph 2<br /><br />0:00:00 Chapter 1<br />0:17:05 Chapter 2<br />0:33:58 Chapter 3<br />0:40:35 Chapter 4<br />Unrelated outro line<br />',
|
|
||||||
audioDuration: 2700,
|
|
||||||
expectedChapters: [
|
|
||||||
{ title: 'Chapter 1', id: 1, start: 0, end: 1025 },
|
|
||||||
{ title: 'Chapter 2', id: 2, start: 1025, end: 2038 },
|
|
||||||
{ title: 'Chapter 3', id: 3, start: 2038, end: 2435 },
|
|
||||||
{ title: 'Chapter 4', id: 4, start: 2435, end: 2700 }
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
testName: 'Should handle descriptions using unix new lines',
|
|
||||||
description: `Introduction text paragraph 1
|
|
||||||
Introduction text paragraph 2
|
|
||||||
0:00:00 Chapter 1
|
|
||||||
0:17:05 Chapter 2
|
|
||||||
0:33:58 Chapter 3
|
|
||||||
0:40:35 Chapter 4
|
|
||||||
Unrelated outro line`,
|
|
||||||
audioDuration: 2700,
|
|
||||||
expectedChapters: [
|
|
||||||
{ title: 'Chapter 1', id: 1, start: 0, end: 1025 },
|
|
||||||
{ title: 'Chapter 2', id: 2, start: 1025, end: 2038 },
|
|
||||||
{ title: 'Chapter 3', id: 3, start: 2038, end: 2435 },
|
|
||||||
{ title: 'Chapter 4', id: 4, start: 2435, end: 2700 }
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
testName: 'Should handle descriptions with no timestamps',
|
|
||||||
description: `Introduction text paragraph 1
|
|
||||||
Introduction text paragraph 2`,
|
|
||||||
audioDuration: 2700,
|
|
||||||
expectedChapters: []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
testName: 'Should handle timestampes in parentheses',
|
|
||||||
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>(00:48) Chatper 1 </p><p>(12:14) Chapter 2 </p><p>(20:56) Chapter 3 </p><p>(27:34) Chapter 4 </p><p>(32:00) Chapter 5 </p><p>(35:16) Chapter 6 </p><p>(41:32) Chapter 7 </p><p>(46:43) Chapter 8</p>',
|
|
||||||
audioDuration: 3060,
|
|
||||||
expectedChapters: [
|
|
||||||
{ title: 'Chatper 1', id: 1, start: 48, end: 734 },
|
|
||||||
{ title: 'Chapter 2', id: 2, start: 734, end: 1256 },
|
|
||||||
{ title: 'Chapter 3', id: 3, start: 1256, end: 1654 },
|
|
||||||
{ title: 'Chapter 4', id: 4, start: 1654, end: 1920 },
|
|
||||||
{ title: 'Chapter 5', id: 5, start: 1920, end: 2116 },
|
|
||||||
{ title: 'Chapter 6', id: 6, start: 2116, end: 2492 },
|
|
||||||
{ title: 'Chapter 7', id: 7, start: 2492, end: 2803 },
|
|
||||||
{ title: 'Chapter 8', id: 8, start: 2803, end: 3060 }
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
testCasesTestingSuccess.forEach(function (testCase) {
|
|
||||||
it(testCase.testName, () => {
|
|
||||||
var chapters = PodcastEpisode.autoGenerateChaptersFromTimestamps(testCase.description, testCase.audioDuration)
|
|
||||||
expect(chapters).to.be.deep.equal(testCase.expectedChapters)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
var testCasesTestingFailure = [
|
|
||||||
{
|
|
||||||
testName: 'Should throw if only one chapter found',
|
|
||||||
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>00:48 Chatper 1 </p>',
|
|
||||||
audioDuration: 1000,
|
|
||||||
expectedError: 'Only one chapter found, treating as invalid description'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
testName: 'Should throw if description is null',
|
|
||||||
description: null,
|
|
||||||
audioDuration: 1000,
|
|
||||||
expectedError: 'Description must not be null'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
testName: 'Should throw if audio duration is null',
|
|
||||||
description: '',
|
|
||||||
audioDuration: null,
|
|
||||||
expectedError: 'Audio duration must not be null'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
testName: 'Should throw if chapter has no title',
|
|
||||||
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>00:48 Chatper 1 </p><p>00:60:12</p>',
|
|
||||||
audioDuration: 1000,
|
|
||||||
expectedError: 'Unable to get chapter title from description'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
testCasesTestingFailure.forEach(function (testCase) {
|
|
||||||
it(testCase.testName, () => {
|
|
||||||
expect(() => {
|
|
||||||
var chapters = PodcastEpisode.autoGenerateChaptersFromTimestamps(testCase.description, testCase.audioDuration)
|
|
||||||
Logger.debug('Chapters', chapters)
|
|
||||||
}).to.throw(testCase.expectedError)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
95
server/utils/parsers/parsePodcastDescriptionForChapters.js
Normal file
95
server/utils/parsers/parsePodcastDescriptionForChapters.js
Normal file
|
|
@ -0,0 +1,95 @@
|
||||||
|
const Logger = require('../../Logger')
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse podcast descriptions for timestamps and generate chapters
|
||||||
|
* The following formats are supports:
|
||||||
|
*
|
||||||
|
* MM:SS Chapter name
|
||||||
|
* HH:MM:SS Chapter name
|
||||||
|
* (HH:MM:SS) Chapter name
|
||||||
|
*
|
||||||
|
* Descriptions have to use <p>, <br> or \n to split up lines in order to be supported
|
||||||
|
*
|
||||||
|
* See test suite for more input examples
|
||||||
|
*
|
||||||
|
* @param {string} podcastDescription
|
||||||
|
* @param {number} audioDurationSecs
|
||||||
|
* @returns {ChapterObject[]}
|
||||||
|
*/
|
||||||
|
module.exports.parse = (podcastDescription, audioDurationSecs) => {
|
||||||
|
if (podcastDescription == null) {
|
||||||
|
throw new Error('Description must not be null')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (audioDurationSecs == null) {
|
||||||
|
throw new Error('Audio duration must not be null')
|
||||||
|
}
|
||||||
|
|
||||||
|
const timestampRegex = /\b(\d{1,2}):(\d{1,2})(?::(\d{1,2}))?\b/
|
||||||
|
const chapterTitleRegex = /\b\d{1,2}:\d{1,2}(?::\d{1,2})?\b(?:\s+|\))(.+)$/
|
||||||
|
const descriptionLineSplitRegex = /\<\s*\/\s*p\s*\>|\<\s*br\s*\s*\/\>|\n/
|
||||||
|
|
||||||
|
var descriptionLines = podcastDescription.split(descriptionLineSplitRegex)
|
||||||
|
var newChapters = []
|
||||||
|
|
||||||
|
for (let i = 0; i < descriptionLines.length; i++) {
|
||||||
|
let line = descriptionLines[i]
|
||||||
|
|
||||||
|
let match = timestampRegex.exec(line)
|
||||||
|
if (match == null) continue
|
||||||
|
|
||||||
|
let first = match[1]
|
||||||
|
let second = match[2]
|
||||||
|
let third = match[3]
|
||||||
|
|
||||||
|
let hours = 0
|
||||||
|
let minutes = 0
|
||||||
|
let seconds = 0
|
||||||
|
|
||||||
|
// If there's three components then we can assume its hh:mm:ss
|
||||||
|
if (first && second && third) {
|
||||||
|
hours = Number(first)
|
||||||
|
minutes = Number(second)
|
||||||
|
seconds = Number(third)
|
||||||
|
} else if (first && second) // otherwise assume mm:ss
|
||||||
|
{
|
||||||
|
minutes = Number(first)
|
||||||
|
seconds = Number(second)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (minutes > 59 || seconds > 59) {
|
||||||
|
throw new Error(`Timestamp contains invalid minutes or seconds field '${minutes}::${seconds}'`)
|
||||||
|
}
|
||||||
|
|
||||||
|
let startTime = seconds + minutes * 60 + hours * 60 * 60
|
||||||
|
if (startTime > audioDurationSecs) {
|
||||||
|
throw new Error(`Chapter found that starts after over audio duration. Duration: ${audioDurationSecs}s - Chapter start ${startTime}s`)
|
||||||
|
}
|
||||||
|
|
||||||
|
let chapterTitleMatch = chapterTitleRegex.exec(line)
|
||||||
|
|
||||||
|
if (chapterTitleMatch == null || chapterTitleMatch.length < 2) {
|
||||||
|
// Unknown chapter state
|
||||||
|
throw new Error(`Unable to get chapter title from description, line ${line}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
let chapter = { title: chapterTitleMatch[1].trim(), id: newChapters.length + 1, start: startTime }
|
||||||
|
|
||||||
|
if (newChapters.length > 0) {
|
||||||
|
newChapters[newChapters.length - 1].end = startTime
|
||||||
|
}
|
||||||
|
|
||||||
|
newChapters.push(chapter)
|
||||||
|
}
|
||||||
|
if (newChapters.length > 0) {
|
||||||
|
newChapters[newChapters.length - 1].end = audioDurationSecs
|
||||||
|
}
|
||||||
|
|
||||||
|
Logger.info(`[PodcastEpisode] Successfully generated ${newChapters.length} chapters`)
|
||||||
|
|
||||||
|
if (newChapters.length == 1) {
|
||||||
|
throw new Error('Only one chapter found, treating as invalid description')
|
||||||
|
}
|
||||||
|
|
||||||
|
return newChapters
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
const chai = require('chai')
|
||||||
|
const sinon = require('sinon')
|
||||||
|
const { expect } = chai
|
||||||
|
|
||||||
|
const { DataTypes, Sequelize } = require('sequelize')
|
||||||
|
const Logger = require('../../../server/Logger')
|
||||||
|
|
||||||
|
const { up, down } = require('../../../server/migrations/v2.34.0-add-auto-generate-podcast-chapters')
|
||||||
|
|
||||||
|
describe('Migration v2.34.0-add-auto-generate-podcast-chapters', () => {
|
||||||
|
let sequelize
|
||||||
|
let queryInterface
|
||||||
|
let loggerInfoStub
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
|
||||||
|
queryInterface = sequelize.getQueryInterface()
|
||||||
|
loggerInfoStub = sinon.stub(Logger, 'info')
|
||||||
|
|
||||||
|
await queryInterface.createTable('podcasts', {
|
||||||
|
id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true },
|
||||||
|
title: { type: DataTypes.STRING, allowNull: false },
|
||||||
|
titleIgnorePrefix: { type: DataTypes.STRING, allowNull: false }
|
||||||
|
})
|
||||||
|
|
||||||
|
await queryInterface.bulkInsert('podcasts', [
|
||||||
|
{ id: 1, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||||
|
{ id: 2, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
sinon.restore()
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('up', () => {
|
||||||
|
it('should add autoGenerateChapters column to podcasts', async () => {
|
||||||
|
await up({ context: { queryInterface, logger: Logger } })
|
||||||
|
|
||||||
|
const [podcasts] = await queryInterface.sequelize.query('SELECT * FROM podcasts')
|
||||||
|
expect(podcasts).to.deep.equal([
|
||||||
|
{ id: 1, autoGenerateChapters: 0, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||||
|
{ id: 2, autoGenerateChapters: 0, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||||
|
])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('down', () => {
|
||||||
|
it('should remove autoGenerateChapters column from podcasts', async () => {
|
||||||
|
await up({ context: { queryInterface, logger: Logger } })
|
||||||
|
try {
|
||||||
|
await down({ context: { queryInterface, logger: Logger } })
|
||||||
|
} catch (error) {
|
||||||
|
console.log(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
const [podcasts] = await queryInterface.sequelize.query('SELECT * FROM podcasts')
|
||||||
|
expect(podcasts).to.deep.equal([
|
||||||
|
{ id: 1, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||||
|
{ id: 2, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||||
|
])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -0,0 +1,132 @@
|
||||||
|
const chai = require('chai')
|
||||||
|
const PodcastEpisode = require('../../../../server/models/PodcastEpisode')
|
||||||
|
const expect = chai.expect
|
||||||
|
const parsePodcastDescriptionForChapters = require('../../../../server/utils/parsers/parsePodcastDescriptionForChapters')
|
||||||
|
|
||||||
|
describe('parsePodcastDescriptionForChapters', () => {
|
||||||
|
var testCasesTestingSuccess = [
|
||||||
|
{
|
||||||
|
testName: 'Should handle descriptions using html paragraphs',
|
||||||
|
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>00:48 Chatper 1 </p><p>12:14 Chapter 2 </p><p>20:56 Chapter 3 </p><p>27:34 Chapter 4 </p><p>32:00 Chapter 5 </p><p>35:16 Chapter 6 </p><p>41:32 Chapter 7 </p><p>46:43 Chapter 8</p>',
|
||||||
|
audioDuration: 3060,
|
||||||
|
expectedChapters: [
|
||||||
|
{ title: 'Chatper 1', id: 1, start: 48, end: 734 },
|
||||||
|
{ title: 'Chapter 2', id: 2, start: 734, end: 1256 },
|
||||||
|
{ title: 'Chapter 3', id: 3, start: 1256, end: 1654 },
|
||||||
|
{ title: 'Chapter 4', id: 4, start: 1654, end: 1920 },
|
||||||
|
{ title: 'Chapter 5', id: 5, start: 1920, end: 2116 },
|
||||||
|
{ title: 'Chapter 6', id: 6, start: 2116, end: 2492 },
|
||||||
|
{ title: 'Chapter 7', id: 7, start: 2492, end: 2803 },
|
||||||
|
{ title: 'Chapter 8', id: 8, start: 2803, end: 3060 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
testName: 'Should handle descriptions using html line breaks',
|
||||||
|
description: '<br>Introduction text paragraph 1<br /><br>Introduction text paragraph 2<br /><br />0:00:00 Chapter 1<br />0:17:05 Chapter 2<br />0:33:58 Chapter 3<br />0:40:35 Chapter 4<br />Unrelated outro line<br />',
|
||||||
|
audioDuration: 2700,
|
||||||
|
expectedChapters: [
|
||||||
|
{ title: 'Chapter 1', id: 1, start: 0, end: 1025 },
|
||||||
|
{ title: 'Chapter 2', id: 2, start: 1025, end: 2038 },
|
||||||
|
{ title: 'Chapter 3', id: 3, start: 2038, end: 2435 },
|
||||||
|
{ title: 'Chapter 4', id: 4, start: 2435, end: 2700 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
testName: 'Should handle descriptions using unix new lines',
|
||||||
|
description: `Introduction text paragraph 1
|
||||||
|
Introduction text paragraph 2
|
||||||
|
0:00:00 Chapter 1
|
||||||
|
0:17:05 Chapter 2
|
||||||
|
0:33:58 Chapter 3
|
||||||
|
0:40:35 Chapter 4
|
||||||
|
Unrelated outro line`,
|
||||||
|
audioDuration: 2700,
|
||||||
|
expectedChapters: [
|
||||||
|
{ title: 'Chapter 1', id: 1, start: 0, end: 1025 },
|
||||||
|
{ title: 'Chapter 2', id: 2, start: 1025, end: 2038 },
|
||||||
|
{ title: 'Chapter 3', id: 3, start: 2038, end: 2435 },
|
||||||
|
{ title: 'Chapter 4', id: 4, start: 2435, end: 2700 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
testName: 'Should handle descriptions with no timestamps',
|
||||||
|
description: `Introduction text paragraph 1
|
||||||
|
Introduction text paragraph 2`,
|
||||||
|
audioDuration: 2700,
|
||||||
|
expectedChapters: []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
testName: 'Should handle timestampes in parentheses',
|
||||||
|
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>(00:48) Chatper 1 </p><p>(12:14) Chapter 2 </p><p>(20:56) Chapter 3 </p><p>(27:34) Chapter 4 </p><p>(32:00) Chapter 5 </p><p>(35:16) Chapter 6 </p><p>(41:32) Chapter 7 </p><p>(46:43) Chapter 8</p>',
|
||||||
|
audioDuration: 3060,
|
||||||
|
expectedChapters: [
|
||||||
|
{ title: 'Chatper 1', id: 1, start: 48, end: 734 },
|
||||||
|
{ title: 'Chapter 2', id: 2, start: 734, end: 1256 },
|
||||||
|
{ title: 'Chapter 3', id: 3, start: 1256, end: 1654 },
|
||||||
|
{ title: 'Chapter 4', id: 4, start: 1654, end: 1920 },
|
||||||
|
{ title: 'Chapter 5', id: 5, start: 1920, end: 2116 },
|
||||||
|
{ title: 'Chapter 6', id: 6, start: 2116, end: 2492 },
|
||||||
|
{ title: 'Chapter 7', id: 7, start: 2492, end: 2803 },
|
||||||
|
{ title: 'Chapter 8', id: 8, start: 2803, end: 3060 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
testCasesTestingSuccess.forEach(function (testCase) {
|
||||||
|
it(testCase.testName, () => {
|
||||||
|
var chapters = parsePodcastDescriptionForChapters.parse(testCase.description, testCase.audioDuration)
|
||||||
|
expect(chapters).to.be.deep.equal(testCase.expectedChapters)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
var testCasesTestingFailure = [
|
||||||
|
{
|
||||||
|
testName: 'Should throw if only one chapter found',
|
||||||
|
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>00:48 Chatper 1 </p>',
|
||||||
|
audioDuration: 1000,
|
||||||
|
expectedError: 'Only one chapter found, treating as invalid description'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
testName: 'Should throw if invalid minutes',
|
||||||
|
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>75:48 Chatper 1 </p>',
|
||||||
|
audioDuration: 1000,
|
||||||
|
expectedError: "Timestamp contains invalid minutes or seconds field '75::48'"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
testName: 'Should throw if invalid minutes',
|
||||||
|
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>00:90 Chatper 1 </p>',
|
||||||
|
audioDuration: 1000,
|
||||||
|
expectedError: "Timestamp contains invalid minutes or seconds field '0::90'"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
testName: 'Should throw if chapter goes over lenght of audio file',
|
||||||
|
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>00:48 Chatper 1 </p><p>01:00:01 Chatper 2 </p>',
|
||||||
|
audioDuration: 3600,
|
||||||
|
expectedError: 'Chapter found that starts after over audio duration'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
testName: 'Should throw if description is null',
|
||||||
|
description: null,
|
||||||
|
audioDuration: 1000,
|
||||||
|
expectedError: 'Description must not be null'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
testName: 'Should throw if audio duration is null',
|
||||||
|
description: '',
|
||||||
|
audioDuration: null,
|
||||||
|
expectedError: 'Audio duration must not be null'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
testName: 'Should throw if chapter has no title',
|
||||||
|
description: '<p>Introduction text paragraph 1</p><p>Introduction text paragraph 2</p><p>00:48 Chatper 1 </p><p>00:30:00</p>',
|
||||||
|
audioDuration: 3600,
|
||||||
|
expectedError: 'Unable to get chapter title from description'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
testCasesTestingFailure.forEach(function (testCase) {
|
||||||
|
it(testCase.testName, () => {
|
||||||
|
expect(() => {
|
||||||
|
parsePodcastDescriptionForChapters.parse(testCase.description, testCase.audioDuration)
|
||||||
|
}).to.throw(testCase.expectedError)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue