From ebff884562cf683311c4f9b568e49838e58497c7 Mon Sep 17 00:00:00 2001 From: Jonathan Baldie Date: Fri, 1 May 2026 21:17:35 +0100 Subject: [PATCH] feat(player): add silence detection and smart speed to local audio player --- client/players/LocalAudioPlayer.js | 62 +++++++ .../smart-speed/SilenceDetectorProcessor.js | 71 ++++++++ client/players/smart-speed/SilenceMap.js | 61 +++++++ .../players/smart-speed/SilenceMap.test.js | 172 ++++++++++++++++++ 4 files changed, 366 insertions(+) create mode 100644 client/players/smart-speed/SilenceDetectorProcessor.js create mode 100644 client/players/smart-speed/SilenceMap.js create mode 100644 test/client/players/smart-speed/SilenceMap.test.js diff --git a/client/players/LocalAudioPlayer.js b/client/players/LocalAudioPlayer.js index 35f45f50..b23d9dc5 100644 --- a/client/players/LocalAudioPlayer.js +++ b/client/players/LocalAudioPlayer.js @@ -1,5 +1,6 @@ import Hls from 'hls.js' import EventEmitter from 'events' +import SilenceMap from './smart-speed/SilenceMap' export default class LocalAudioPlayer extends EventEmitter { constructor(ctx) { @@ -25,6 +26,10 @@ export default class LocalAudioPlayer extends EventEmitter { this.audioSourceNode = null this.usingWebAudio = false + this.silenceMap = new SilenceMap() + this.silenceDetectorNode = null + this.enableSmartSpeed = false + this.initialize() } @@ -92,6 +97,51 @@ export default class LocalAudioPlayer extends EventEmitter { } } + async initSilenceDetector() { + if (!this.usingWebAudio || !this.audioContext) return + if (this.silenceDetectorNode) return + + try { + await this.audioContext.audioWorklet.addModule('/client/players/smart-speed/SilenceDetectorProcessor.js') + this.silenceDetectorNode = new AudioWorkletNode(this.audioContext, 'silence-detector') + + this.silenceDetectorNode.port.onmessage = (event) => { + const msg = event.data + if (msg.type === 'silence-start') { + this._silenceStartTime = msg.time + } else if (msg.type === 'silence-end') { + if (this._silenceStartTime !== null) { + this.silenceMap.addRegion(this._silenceStartTime, msg.time) + this._silenceStartTime = null + } + } + } + + this.audioSourceNode.disconnect() + this.audioSourceNode.connect(this.silenceDetectorNode) + this.silenceDetectorNode.connect(this.audioContext.destination) + + this._silenceStartTime = null + console.log('[LocalPlayer] Silence detector initialised') + } catch (err) { + console.warn('[LocalPlayer] Failed to initialise silence detector', err) + this.silenceDetectorNode = null + } + } + + destroySilenceDetector() { + if (this.silenceDetectorNode) { + try { + this.silenceDetectorNode.disconnect() + } catch (err) { + // Ignore disconnect errors + } + this.silenceDetectorNode = null + } + this.silenceMap.reset() + this._silenceStartTime = null + } + evtPlay() { this.emit('stateChange', 'PLAYING') } @@ -137,6 +187,7 @@ export default class LocalAudioPlayer extends EventEmitter { } destroy() { + this.destroySilenceDetector() this.destroyHlsInstance() this.destroyWebAudio() if (this.player) { @@ -239,6 +290,7 @@ export default class LocalAudioPlayer extends EventEmitter { loadCurrentTrack() { if (!this.currentTrack) return + this.silenceMap.reset() // When direct play track is loaded current time needs to be set this.trackStartTime = Math.max(0, this.startTime - (this.currentTrack.startOffset || 0)) this.player.src = this.currentTrack.relativeContentUrl @@ -319,9 +371,19 @@ export default class LocalAudioPlayer extends EventEmitter { this.player.playbackRate = playbackRate } + async setSmartSpeed(enabled) { + this.enableSmartSpeed = enabled + if (enabled && this.usingWebAudio) { + await this.initSilenceDetector() + } else { + this.destroySilenceDetector() + } + } + seek(time, playWhenReady) { if (!this.player) return + this.silenceMap.reset() this.playWhenReady = playWhenReady if (this.isHlsTranscode) { diff --git a/client/players/smart-speed/SilenceDetectorProcessor.js b/client/players/smart-speed/SilenceDetectorProcessor.js new file mode 100644 index 00000000..fd1fe190 --- /dev/null +++ b/client/players/smart-speed/SilenceDetectorProcessor.js @@ -0,0 +1,71 @@ +const SPEAKING = 0 +const SILENCE = 1 +const CANDIDATE = 2 + +const DEBOUNCE_MS = 200 +const RMS_REPORT_INTERVAL = 10 + +class SilenceDetectorProcessor extends AudioWorkletProcessor { + constructor() { + super() + this.state = SPEAKING + this.silenceThreshold = -40 + this.candidateStartSample = 0 + this.sampleRate = sampleRate + this.blockCount = 0 + + this.port.onmessage = (event) => { + const msg = event.data + if (msg.type === 'set-threshold') { + this.silenceThreshold = msg.value + } + } + } + + process(inputs) { + const input = inputs[0] + if (!input || !input.length) return true + + const channel = input[0] + if (!channel) return true + + let sum = 0 + for (let i = 0; i < channel.length; i++) { + sum += channel[i] * channel[i] + } + const rms = Math.sqrt(sum / channel.length) + const dbfs = rms === 0 ? -Infinity : 20 * Math.log10(rms) + + this.blockCount++ + + if (dbfs < this.silenceThreshold) { + if (this.state === SPEAKING) { + this.candidateStartSample = currentFrame + this.state = CANDIDATE + } else if (this.state === CANDIDATE) { + const elapsedMs = ((currentFrame - this.candidateStartSample) / this.sampleRate) * 1000 + if (elapsedMs >= DEBOUNCE_MS) { + this.state = SILENCE + const silenceStartTime = (this.candidateStartSample / this.sampleRate) * 1000 + this.port.postMessage({ type: 'silence-start', time: silenceStartTime }) + } + } + } else { + if (this.state === SILENCE) { + const currentTime = (currentFrame / this.sampleRate) * 1000 + this.port.postMessage({ type: 'silence-end', time: currentTime }) + } + if (this.state !== SPEAKING) { + this.state = SPEAKING + } + } + + if (this.blockCount % RMS_REPORT_INTERVAL === 0) { + this.port.postMessage({ type: 'rms', value: dbfs }) + } + + return true + } +} + +registerProcessor('silence-detector', SilenceDetectorProcessor) diff --git a/client/players/smart-speed/SilenceMap.js b/client/players/smart-speed/SilenceMap.js new file mode 100644 index 00000000..5fbe7aba --- /dev/null +++ b/client/players/smart-speed/SilenceMap.js @@ -0,0 +1,61 @@ +class SilenceMap { + constructor() { + this._regions = [] + } + + get regionCount() { + return this._regions.length + } + + getRegions() { + return [...this._regions] + } + + addRegion(startMs, endMs) { + if (typeof startMs !== 'number' || typeof endMs !== 'number') return + if (startMs < 0 || endMs < 0) return + if (endMs <= startMs) return + + const newRegion = { start: startMs, end: endMs } + const merged = [] + let inserted = false + + for (const region of this._regions) { + if (newRegion.start <= region.end + 10 && newRegion.end >= region.start - 10) { + newRegion.start = Math.min(newRegion.start, region.start) + newRegion.end = Math.max(newRegion.end, region.end) + } else if (!inserted && region.start > newRegion.end) { + merged.push(newRegion) + merged.push(region) + inserted = true + } else { + merged.push(region) + } + } + + if (!inserted) { + merged.push(newRegion) + } + + this._regions = merged + } + + getCompressedOffset(atTimeMs, ratio) { + if (!ratio || ratio <= 1) return 0 + let saved = 0 + for (const region of this._regions) { + if (atTimeMs <= region.start) break + const silenceStart = region.start + const silenceEnd = Math.min(region.end, atTimeMs) + const silenceDuration = silenceEnd - silenceStart + saved += silenceDuration * (1 - 1 / ratio) + } + return saved + } + + reset() { + this._regions = [] + } +} + +module.exports = SilenceMap diff --git a/test/client/players/smart-speed/SilenceMap.test.js b/test/client/players/smart-speed/SilenceMap.test.js new file mode 100644 index 00000000..224081ea --- /dev/null +++ b/test/client/players/smart-speed/SilenceMap.test.js @@ -0,0 +1,172 @@ +const chai = require('chai') +const expect = chai.expect +const SilenceMap = require('../../../../client/players/smart-speed/SilenceMap') + +describe('SilenceMap', () => { + let map + + beforeEach(() => { + map = new SilenceMap() + }) + + describe('Basic operations', () => { + it('1. Empty map returns 0 regions', () => { + expect(map.regionCount).to.equal(0) + expect(map.getRegions()).to.deep.equal([]) + }) + + it('2. Single region add/get', () => { + map.addRegion(100, 300) + expect(map.regionCount).to.equal(1) + expect(map.getRegions()).to.deep.equal([{ start: 100, end: 300 }]) + }) + + it('3. Overlapping regions merge correctly', () => { + map.addRegion(100, 300) + map.addRegion(200, 400) + expect(map.regionCount).to.equal(1) + expect(map.getRegions()).to.deep.equal([{ start: 100, end: 400 }]) + }) + + it('4. Non-overlapping regions stay separate', () => { + map.addRegion(100, 200) + map.addRegion(300, 400) + expect(map.regionCount).to.equal(2) + expect(map.getRegions()).to.deep.equal([ + { start: 100, end: 200 }, + { start: 300, end: 400 } + ]) + }) + + it('5. Adjacent regions (gap < 10ms) merge', () => { + map.addRegion(100, 200) + map.addRegion(205, 300) + expect(map.regionCount).to.equal(1) + expect(map.getRegions()).to.deep.equal([{ start: 100, end: 300 }]) + }) + + it('6. Three+ overlapping regions merge into one', () => { + map.addRegion(100, 300) + map.addRegion(200, 400) + map.addRegion(350, 500) + expect(map.regionCount).to.equal(1) + expect(map.getRegions()).to.deep.equal([{ start: 100, end: 500 }]) + }) + }) + + describe('getCompressedOffset', () => { + it('7. getCompressedOffset(0) returns 0', () => { + map.addRegion(100, 300) + expect(map.getCompressedOffset(0, 2)).to.equal(0) + }) + + it('8. getCompressedOffset at region boundary', () => { + map.addRegion(100, 300) + // At time 100ms (start of region), no compression has happened yet + expect(map.getCompressedOffset(100, 2)).to.equal(0) + }) + + it('9. getCompressedOffset inside region', () => { + map.addRegion(100, 300) + // At time 200ms (100ms into a 200ms region), with ratio 2x: + // 100ms of silence consumed, compressed to 50ms, saving 50ms + expect(map.getCompressedOffset(200, 2)).to.equal(50) + }) + + it('10. getCompressedOffset after region with ratio 2x', () => { + map.addRegion(100, 300) + // At time 500ms (after the 200ms region), with ratio 2x: + // 200ms of silence, compressed to 100ms, saving 100ms + expect(map.getCompressedOffset(500, 2)).to.equal(100) + }) + + it('11. getCompressedOffset with multiple regions', () => { + map.addRegion(100, 200) // 100ms region + map.addRegion(400, 600) // 200ms region + // At time 700ms, with ratio 2x: + // Region 1: 100ms silence → 50ms, saving 50ms + // Region 2: 200ms silence → 100ms, saving 100ms + // Total saved: 150ms + expect(map.getCompressedOffset(700, 2)).to.equal(150) + }) + }) + + describe('Reset and state', () => { + it('12. reset() clears everything', () => { + map.addRegion(100, 300) + map.addRegion(400, 600) + map.reset() + expect(map.regionCount).to.equal(0) + expect(map.getRegions()).to.deep.equal([]) + }) + + it('13. Regions always sorted by start time', () => { + map.addRegion(500, 600) + map.addRegion(100, 200) + map.addRegion(300, 400) + const regions = map.getRegions() + expect(regions[0].start).to.equal(100) + expect(regions[1].start).to.equal(300) + expect(regions[2].start).to.equal(500) + }) + }) + + describe('Validation', () => { + it('14. Invalid region (end <= start) is rejected', () => { + map.addRegion(300, 100) + expect(map.regionCount).to.equal(0) + }) + + it('15. Region at time 0', () => { + map.addRegion(0, 100) + expect(map.regionCount).to.equal(1) + expect(map.getRegions()).to.deep.equal([{ start: 0, end: 100 }]) + }) + + it('16. Very large time values (24 hours)', () => { + map.addRegion(86400000, 86401000) + expect(map.regionCount).to.equal(1) + expect(map.getRegions()).to.deep.equal([{ start: 86400000, end: 86401000 }]) + expect(map.getCompressedOffset(86402000, 2)).to.equal(500) + }) + }) + + describe('Edge cases', () => { + it('17. Rapid addRegion calls (1000 regions)', () => { + for (let i = 0; i < 1000; i++) { + map.addRegion(i * 100, i * 100 + 50) + } + expect(map.regionCount).to.equal(1000) + }) + + it('18. Region with identical start and end is rejected', () => { + map.addRegion(100, 100) + expect(map.regionCount).to.equal(0) + }) + + it('19. Region with negative values is rejected', () => { + map.addRegion(-100, 100) + expect(map.regionCount).to.equal(0) + }) + + it('20. Multiple resets do not error', () => { + map.addRegion(100, 300) + map.reset() + map.reset() + map.reset() + expect(map.regionCount).to.equal(0) + }) + + it('21. getCompressedOffset with ratio 1.0 (no compression)', () => { + map.addRegion(100, 300) + // ratio 1.0 means no speedup, so no time saved + expect(map.getCompressedOffset(500, 1.0)).to.equal(0) + }) + + it('22. getCompressedOffset with ratio 5.0 (aggressive)', () => { + map.addRegion(100, 300) + // 200ms region at 5x: compressed to 40ms, saving 160ms + expect(map.getCompressedOffset(500, 5.0)).to.equal(160) + }) + }) +})