mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2026-05-13 06:51:29 +00:00
Redesign Smart Speed to dynamically adjust playbackRate instead of dropping samples, fix TimeMapper bugs by mapping audioContext to media time, and prevent SilenceMap memory leak by capping regions
This commit is contained in:
Jonathan Baldie
parent
fa2460868e
commit
545c77a2dc
4 changed files with 32 additions and 314 deletions
|
|
@ -29,7 +29,6 @@ export default class LocalAudioPlayer extends EventEmitter {
|
|||
|
||||
this.silenceMap = new SilenceMap()
|
||||
this.silenceDetectorNode = null
|
||||
this.silenceCompressorNode = null
|
||||
this.timeMapper = new TimeMapper([], 1.0)
|
||||
this.smartSpeedRatio = 2.0
|
||||
this.enableSmartSpeed = false
|
||||
|
|
@ -102,10 +101,8 @@ export default class LocalAudioPlayer extends EventEmitter {
|
|||
}
|
||||
|
||||
updateSmartSpeedRegions() {
|
||||
if (this.silenceCompressorNode) {
|
||||
this.silenceCompressorNode.port.postMessage({ type: 'set-regions', regions: this.silenceMap.getRegions() })
|
||||
}
|
||||
this.timeMapper = new TimeMapper(this.silenceMap.getRegions(), this.smartSpeedRatio)
|
||||
this.emit('timeSaved', this.timeMapper.totalTimeSaved())
|
||||
}
|
||||
|
||||
async initSilenceDetector() {
|
||||
|
|
@ -114,24 +111,27 @@ export default class LocalAudioPlayer extends EventEmitter {
|
|||
|
||||
try {
|
||||
await this.audioContext.audioWorklet.addModule('/client/players/smart-speed/SilenceDetectorProcessor.js')
|
||||
await this.audioContext.audioWorklet.addModule('/client/players/smart-speed/SilenceCompressorProcessor.js')
|
||||
this.silenceDetectorNode = new AudioWorkletNode(this.audioContext, 'silence-detector')
|
||||
this.silenceCompressorNode = new AudioWorkletNode(this.audioContext, 'silence-compressor')
|
||||
this.silenceCompressorNode.port.postMessage({ type: 'set-ratio', value: this.smartSpeedRatio })
|
||||
this.silenceCompressorNode.port.onmessage = (event) => {
|
||||
const msg = event.data
|
||||
if (msg.type === 'time-saved') {
|
||||
this.emit('timeSaved', msg.ms)
|
||||
}
|
||||
}
|
||||
|
||||
this.silenceDetectorNode.port.onmessage = (event) => {
|
||||
const msg = event.data
|
||||
if (msg.type === 'silence-start') {
|
||||
this._silenceStartTime = msg.time
|
||||
// Map AudioContext time to Media time
|
||||
const delayMs = this.audioContext.currentTime * 1000 - msg.time
|
||||
this._silenceStartTime = this.player.currentTime * 1000 - delayMs
|
||||
|
||||
// Dynamically increase playback rate
|
||||
if (this.enableSmartSpeed) {
|
||||
this.player.playbackRate = this.defaultPlaybackRate * this.smartSpeedRatio
|
||||
}
|
||||
} else if (msg.type === 'silence-end') {
|
||||
if (this.enableSmartSpeed) {
|
||||
this.player.playbackRate = this.defaultPlaybackRate
|
||||
}
|
||||
if (this._silenceStartTime !== null) {
|
||||
this.silenceMap.addRegion(this._silenceStartTime, msg.time)
|
||||
const delayMs = this.audioContext.currentTime * 1000 - msg.time
|
||||
const silenceEndTime = this.player.currentTime * 1000 - delayMs
|
||||
this.silenceMap.addRegion(this._silenceStartTime, silenceEndTime)
|
||||
this._silenceStartTime = null
|
||||
this.updateSmartSpeedRegions()
|
||||
}
|
||||
|
|
@ -140,8 +140,7 @@ export default class LocalAudioPlayer extends EventEmitter {
|
|||
|
||||
this.audioSourceNode.disconnect()
|
||||
this.audioSourceNode.connect(this.silenceDetectorNode)
|
||||
this.silenceDetectorNode.connect(this.silenceCompressorNode)
|
||||
this.silenceCompressorNode.connect(this.audioContext.destination)
|
||||
this.silenceDetectorNode.connect(this.audioContext.destination)
|
||||
|
||||
this._silenceStartTime = null
|
||||
console.log('[LocalPlayer] Silence detector initialised')
|
||||
|
|
@ -160,15 +159,14 @@ export default class LocalAudioPlayer extends EventEmitter {
|
|||
}
|
||||
this.silenceDetectorNode = null
|
||||
}
|
||||
if (this.silenceCompressorNode) {
|
||||
try {
|
||||
this.silenceCompressorNode.disconnect()
|
||||
} catch (err) {}
|
||||
this.silenceCompressorNode = null
|
||||
}
|
||||
this.silenceMap.reset()
|
||||
this.updateSmartSpeedRegions()
|
||||
this._silenceStartTime = null
|
||||
|
||||
// Reset playback rate in case we were in the middle of a silence region
|
||||
if (this.player && this.player.playbackRate !== this.defaultPlaybackRate) {
|
||||
this.player.playbackRate = this.defaultPlaybackRate
|
||||
}
|
||||
}
|
||||
|
||||
evtPlay() {
|
||||
|
|
@ -388,11 +386,6 @@ export default class LocalAudioPlayer extends EventEmitter {
|
|||
var currentTrackOffset = this.currentTrack.startOffset || 0
|
||||
if (!this.player) return 0
|
||||
|
||||
if (this.enableSmartSpeed) {
|
||||
var audioMs = this.player.currentTime * 1000
|
||||
var wallMs = this.timeMapper.audioToWallClock(audioMs)
|
||||
return currentTrackOffset + (wallMs / 1000)
|
||||
}
|
||||
return currentTrackOffset + this.player.currentTime
|
||||
}
|
||||
|
||||
|
|
@ -420,16 +413,16 @@ export default class LocalAudioPlayer extends EventEmitter {
|
|||
seek(time, playWhenReady) {
|
||||
if (!this.player) return
|
||||
|
||||
// Map wall-clock seek time to audio time before resetting regions
|
||||
var mappedTime = time
|
||||
if (this.enableSmartSpeed && time >= (this.currentTrack.startOffset || 0) && time <= (this.currentTrack.startOffset || 0) + (this.currentTrack.duration || Infinity)) {
|
||||
var offsetTime = mappedTime - (this.currentTrack.startOffset || 0)
|
||||
mappedTime = (this.currentTrack.startOffset || 0) + (this.timeMapper.wallClockToAudio(offsetTime * 1000) / 1000)
|
||||
}
|
||||
|
||||
this.silenceMap.reset()
|
||||
this.updateSmartSpeedRegions()
|
||||
this.playWhenReady = playWhenReady
|
||||
|
||||
// Reset playback rate in case we were in a silence region
|
||||
if (this.enableSmartSpeed && this.player.playbackRate !== this.defaultPlaybackRate) {
|
||||
this.player.playbackRate = this.defaultPlaybackRate
|
||||
}
|
||||
|
||||
if (this.isHlsTranscode) {
|
||||
// Seeking HLS stream
|
||||
|
|
|
|||
|
|
@ -1,122 +0,0 @@
|
|||
class SilenceCompressorProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super()
|
||||
this.regions = []
|
||||
this.ratio = 1.0
|
||||
this.totalCompressedMs = 0
|
||||
this.rampDurationSec = 0.005 // 5ms
|
||||
|
||||
this.port.onmessage = (event) => {
|
||||
const msg = event.data
|
||||
if (msg.type === 'set-regions') {
|
||||
this.regions = msg.regions.filter(r => (r.end - r.start) >= 200)
|
||||
} else if (msg.type === 'set-ratio') {
|
||||
this.ratio = msg.value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
getActiveRegion(timeMs) {
|
||||
for (const r of this.regions) {
|
||||
if (timeMs >= r.start && timeMs <= r.end) return r
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
calculateRampGain(timeMs, region) {
|
||||
const rampMs = this.rampDurationSec * 1000
|
||||
|
||||
// Entry ramp (0 -> 1)
|
||||
if (timeMs - region.start < rampMs) {
|
||||
return (timeMs - region.start) / rampMs
|
||||
}
|
||||
|
||||
// Exit ramp (1 -> 0)
|
||||
if (region.end - timeMs < rampMs) {
|
||||
return (region.end - timeMs) / rampMs
|
||||
}
|
||||
|
||||
return 1.0
|
||||
}
|
||||
|
||||
process(inputs, outputs, parameters) {
|
||||
const input = inputs[0]
|
||||
const output = outputs[0]
|
||||
|
||||
if (!input || !input.length || !output || !output.length) return true
|
||||
|
||||
const numChannels = input.length
|
||||
const numFrames = input[0].length
|
||||
const sampleRateC = typeof sampleRate !== 'undefined' ? sampleRate : 48000
|
||||
// Use currentTime if available, otherwise fallback to 0 (for tests)
|
||||
const currentTimeSec = typeof currentTime !== 'undefined' ? currentTime : 0
|
||||
|
||||
let outputIndex = 0
|
||||
let inputIndex = 0
|
||||
let savedSecThisBlock = 0
|
||||
|
||||
while (inputIndex < numFrames) {
|
||||
const sampleTimeSec = currentTimeSec + (inputIndex / sampleRateC)
|
||||
const sampleTimeMs = sampleTimeSec * 1000
|
||||
|
||||
const region = this.getActiveRegion(sampleTimeMs)
|
||||
|
||||
let step = 1.0
|
||||
let rampGain = 1.0
|
||||
|
||||
if (region && this.ratio > 1.0) {
|
||||
step = this.ratio
|
||||
rampGain = this.calculateRampGain(sampleTimeMs, region)
|
||||
}
|
||||
|
||||
// If taking this step exceeds the input buffer, we must stop
|
||||
if (inputIndex >= numFrames) break
|
||||
|
||||
const intIndex = Math.floor(inputIndex)
|
||||
const frac = inputIndex - intIndex
|
||||
|
||||
for (let c = 0; c < numChannels; c++) {
|
||||
const inChannel = input[c]
|
||||
const outChannel = output[c]
|
||||
|
||||
let sample = inChannel[intIndex]
|
||||
if (frac > 0 && intIndex + 1 < numFrames) {
|
||||
sample = sample + frac * (inChannel[intIndex + 1] - sample)
|
||||
}
|
||||
|
||||
if (outputIndex < numFrames) {
|
||||
outChannel[outputIndex] = sample * rampGain
|
||||
}
|
||||
}
|
||||
|
||||
inputIndex += step
|
||||
outputIndex += 1
|
||||
|
||||
if (step > 1.0) {
|
||||
savedSecThisBlock += (step - 1.0) / sampleRateC
|
||||
}
|
||||
}
|
||||
|
||||
// Fill the rest of the output buffer with 0s if we compressed
|
||||
for (let c = 0; c < numChannels; c++) {
|
||||
for (let i = outputIndex; i < numFrames; i++) {
|
||||
output[c][i] = 0
|
||||
}
|
||||
}
|
||||
|
||||
if (savedSecThisBlock > 0) {
|
||||
this.totalCompressedMs += savedSecThisBlock * 1000
|
||||
this.port.postMessage({ type: 'time-saved', ms: this.totalCompressedMs })
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof registerProcessor !== 'undefined') {
|
||||
registerProcessor('silence-compressor', SilenceCompressorProcessor)
|
||||
}
|
||||
|
||||
if (typeof module !== 'undefined') {
|
||||
module.exports = SilenceCompressorProcessor
|
||||
}
|
||||
|
|
@ -38,6 +38,12 @@ class SilenceMap {
|
|||
}
|
||||
|
||||
this._regions = merged
|
||||
|
||||
// Cap the number of regions to prevent memory leaks for long audiobooks
|
||||
// Assuming each region is ~1 second, 5000 regions is over an hour of silence
|
||||
if (this._regions.length > 5000) {
|
||||
this._regions = this._regions.slice(-5000)
|
||||
}
|
||||
}
|
||||
|
||||
getCompressedOffset(atTimeMs, ratio) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue