From b37a863c0afd8ad4c933d7487fea9ca06cea0263 Mon Sep 17 00:00:00 2001
From: mfcar <marcosfcar@gmail.com>
Date: Sat, 4 May 2024 11:00:30 +0100
Subject: [PATCH 1/9] Initial transcription support

---
 .../components/app/MediaPlayerContainer.vue   | 21 +++++-
 client/components/player/PlayerUi.vue         | 12 ++-
 .../components/player/TranscriptionLine.vue   | 29 +++++++
 client/components/player/TranscriptionUi.vue  | 40 ++++++++++
 client/players/AudioTrack.js                  | 67 ++++++++++-------
 client/players/LocalAudioPlayer.js            | 31 +++++++-
 client/strings/en-us.json                     |  3 +-
 server/controllers/LibraryItemController.js   | 75 ++++++++++++-------
 server/objects/files/AudioTrack.js            |  5 +-
 server/routers/ApiRouter.js                   |  7 +-
 10 files changed, 227 insertions(+), 63 deletions(-)
 create mode 100644 client/components/player/TranscriptionLine.vue
 create mode 100644 client/components/player/TranscriptionUi.vue

diff --git a/client/components/app/MediaPlayerContainer.vue b/client/components/app/MediaPlayerContainer.vue
index 3c4b1d35b..c34e8595b 100644
--- a/client/components/app/MediaPlayerContainer.vue
+++ b/client/components/app/MediaPlayerContainer.vue
@@ -1,5 +1,6 @@
 <template>
-  <div v-if="streamLibraryItem" id="mediaPlayerContainer" class="w-full fixed bottom-0 left-0 right-0 h-48 lg:h-40 z-50 bg-primary px-2 lg:px-4 pb-1 lg:pb-4 pt-2">
+    <div v-if="streamLibraryItem" id="mediaPlayerContainer"  class="w-full fixed bottom-0 left-0 right-0 z-50 bg-primary px-2 lg:px-4 pb-1 lg:pb-4 pt-2"
+         :class="[showTranscriptionUi ? 'h-64' : 'h-48', showTranscriptionUi ? 'lg:h-64' : 'lg:h-40']">
     <div id="videoDock" />
     <div class="absolute left-2 top-2 lg:left-4 cursor-pointer">
       <covers-book-cover expand-on-click :library-item="streamLibraryItem" :width="bookCoverWidth" :book-cover-aspect-ratio="coverAspectRatio" />
@@ -41,6 +42,7 @@
       :sleep-timer-set="sleepTimerSet"
       :sleep-timer-remaining="sleepTimerRemaining"
       :is-podcast="isPodcast"
+      :transcription-enabled="showTranscriptionUi"
       @playPause="playPause"
       @jumpForward="jumpForward"
       @jumpBackward="jumpBackward"
@@ -51,8 +53,11 @@
       @showBookmarks="showBookmarks"
       @showSleepTimer="showSleepTimerModal = true"
       @showPlayerQueueItems="showPlayerQueueItemsModal = true"
+      @showTranscription="showTranscriptionUi = !showTranscriptionUi"
     />
 
+    <transcription-ui v-if="showTranscriptionUi"></transcription-ui>
+
     <modals-bookmarks-modal v-model="showBookmarksModal" :bookmarks="bookmarks" :current-time="bookmarkCurrentTime" :library-item-id="libraryItemId" @select="selectBookmark" />
 
     <modals-sleep-timer-modal v-model="showSleepTimerModal" :timer-set="sleepTimerSet" :timer-time="sleepTimerTime" :remaining="sleepTimerRemaining" @set="setSleepTimer" @cancel="cancelSleepTimer" @increment="incrementSleepTimer" @decrement="decrementSleepTimer" />
@@ -63,8 +68,10 @@
 
 <script>
 import PlayerHandler from '@/players/PlayerHandler'
+import TranscriptionUi from "@/components/player/TranscriptionUi.vue";
 
 export default {
+  components: {TranscriptionUi},
   data() {
     return {
       playerHandler: new PlayerHandler(this),
@@ -76,6 +83,7 @@ export default {
       currentTime: 0,
       showSleepTimerModal: false,
       showPlayerQueueItemsModal: false,
+      showTranscriptionUi: false,
       sleepTimerSet: false,
       sleepTimerTime: 0,
       sleepTimerRemaining: 0,
@@ -86,6 +94,17 @@ export default {
       coverAspectRatio: 1
     }
   },
+  watch: {
+    'playerHandler.playerState': function (newVal, oldVal) {
+      // Refresh the transcription UI when the player is changed
+      if (newVal === 'LOADED') {
+        this.showTranscriptionUi = false;
+        this.$nextTick(() => {
+          this.showTranscriptionUi = true;
+        });
+      }
+    },
+  },
   computed: {
     isSquareCover() {
       return this.coverAspectRatio === 1
diff --git a/client/components/player/PlayerUi.vue b/client/components/player/PlayerUi.vue
index 68d3f947d..ac3160e45 100644
--- a/client/components/player/PlayerUi.vue
+++ b/client/components/player/PlayerUi.vue
@@ -36,6 +36,13 @@
           </button>
         </ui-tooltip>
 
+        <ui-tooltip direction="top" :text="$strings.LabelViewTranscription">
+          <button :aria-label="$strings.LabelViewTranscription" class="outline-none text-gray-300 mx-1 lg:mx-2 hover:text-white" @mousedown.prevent @mouseup.prevent @click.stop="$emit('showTranscription')">
+            <span v-if="!transcriptionEnabled" class="material-icons text-2xl">subtitles</span>
+            <span v-else  class="material-icons text-2xl text-warning">subtitles</span>
+          </button>
+        </ui-tooltip>
+
         <ui-tooltip v-if="chapters.length" direction="top" :text="useChapterTrack ? $strings.LabelUseFullTrack : $strings.LabelUseChapterTrack">
           <button :aria-label="useChapterTrack ? $strings.LabelUseFullTrack : $strings.LabelUseChapterTrack" class="text-gray-300 mx-1 lg:mx-2 hover:text-white" @mousedown.prevent @mouseup.prevent @click.stop="setUseChapterTrack">
             <span class="material-icons text-2xl sm:text-3xl transform transition-transform" :class="useChapterTrack ? 'rotate-180' : ''">timelapse</span>
@@ -78,7 +85,8 @@ export default {
     },
     sleepTimerSet: Boolean,
     sleepTimerRemaining: Number,
-    isPodcast: Boolean
+    isPodcast: Boolean,
+    transcriptionEnabled: Boolean
   },
   data() {
     return {
@@ -368,4 +376,4 @@ export default {
     left: 100%;
   }
 }
-</style>
\ No newline at end of file
+</style>
diff --git a/client/components/player/TranscriptionLine.vue b/client/components/player/TranscriptionLine.vue
new file mode 100644
index 000000000..ff074977e
--- /dev/null
+++ b/client/components/player/TranscriptionLine.vue
@@ -0,0 +1,29 @@
+<template>
+  <div :class="{ 'text-warning': isActive }">
+    <div v-html="cue.text"></div>
+  </div>
+</template>
+
+<script>
+export default {
+  props: {
+    cue: VTTCue
+  },
+  data() {
+    return {
+      isActive: false
+    };
+  },
+  created() {
+    this.cue.onenter = () => (this.isActive = true);
+    this.cue.onexit = () => (this.isActive = false);
+  },
+  watch: {
+    isActive(newVal) {
+      if (newVal) {
+        this.$el.scrollIntoView({behavior: 'smooth'});
+      }
+    }
+  }
+};
+</script>
diff --git a/client/components/player/TranscriptionUi.vue b/client/components/player/TranscriptionUi.vue
new file mode 100644
index 000000000..73d807f2b
--- /dev/null
+++ b/client/components/player/TranscriptionUi.vue
@@ -0,0 +1,40 @@
+<template>
+  <div id="transcription-panel">
+    <transcription-line
+      v-for="(cue, index) in cues"
+      :key="index"
+      :cue="cue"
+      ref="transcriptionLine + index"
+    ></transcription-line>
+  </div>
+</template>
+
+<script>
+import TranscriptionLine from "./TranscriptionLine.vue";
+
+export default {
+  components: {
+    TranscriptionLine
+  },
+  data() {
+    return {
+      cues: [],
+    };
+  },
+  mounted() {
+    this.init()
+  },
+  methods: {
+    init() {
+      const trackElement = document.getElementById("transcription-track");
+      this.cues = trackElement.track.cues;
+    },
+  },
+};
+</script>
+<style>
+#transcription-panel {
+  max-height: 75px;
+  overflow-y: auto;
+}
+</style>
diff --git a/client/players/AudioTrack.js b/client/players/AudioTrack.js
index f364dad89..2146c3e09 100644
--- a/client/players/AudioTrack.js
+++ b/client/players/AudioTrack.js
@@ -1,32 +1,43 @@
 export default class AudioTrack {
-  constructor(track, userToken) {
-    this.index = track.index || 0
-    this.startOffset = track.startOffset || 0 // Total time of all previous tracks
-    this.duration = track.duration || 0
-    this.title = track.title || ''
-    this.contentUrl = track.contentUrl || null
-    this.mimeType = track.mimeType
-    this.metadata = track.metadata || {}
+    constructor(track, userToken) {
+        this.index = track.index || 0
+        this.startOffset = track.startOffset || 0 // Total time of all previous tracks
+        this.duration = track.duration || 0
+        this.title = track.title || ''
+        this.contentUrl = track.contentUrl || null
+        this.transcriptUrl = track.transcriptUrl || null
+        this.mimeType = track.mimeType
+        this.metadata = track.metadata || {}
 
-    this.userToken = userToken
-  }
-
-  get fullContentUrl() {
-    if (!this.contentUrl || this.contentUrl.startsWith('http')) return this.contentUrl
-
-    if (process.env.NODE_ENV === 'development') {
-      return `${process.env.serverUrl}${this.contentUrl}?token=${this.userToken}`
-    }
-    return `${window.location.origin}${this.contentUrl}?token=${this.userToken}`
-  }
-
-  get relativeContentUrl() {
-    if (!this.contentUrl || this.contentUrl.startsWith('http')) return this.contentUrl
-
-    if (process.env.NODE_ENV === 'development') {
-      return `${process.env.serverUrl}${this.contentUrl}?token=${this.userToken}`
+        this.userToken = userToken
     }
 
-    return this.contentUrl + `?token=${this.userToken}`
-  }
-}
\ No newline at end of file
+    get fullContentUrl() {
+        if (!this.contentUrl || this.contentUrl.startsWith('http')) return this.contentUrl
+
+        if (process.env.NODE_ENV === 'development') {
+            return `${process.env.serverUrl}${this.contentUrl}?token=${this.userToken}`
+        }
+        return `${window.location.origin}${this.contentUrl}?token=${this.userToken}`
+    }
+
+    get relativeContentUrl() {
+        if (!this.contentUrl || this.contentUrl.startsWith('http')) return this.contentUrl
+
+        if (process.env.NODE_ENV === 'development') {
+            return `${process.env.serverUrl}${this.contentUrl}?token=${this.userToken}`
+        }
+
+        return this.contentUrl + `?token=${this.userToken}`
+    }
+
+    get relativeTranscriptionUrl() {
+        if (!this.transcriptUrl || this.transcriptUrl.startsWith('http')) return this.transcriptUrl
+
+        if (process.env.NODE_ENV === 'development') {
+            return `${process.env.serverUrl}${this.transcriptUrl}?token=${this.userToken}`
+        }
+
+        return this.transcriptUrl + `?token=${this.userToken}`
+    }
+}
diff --git a/client/players/LocalAudioPlayer.js b/client/players/LocalAudioPlayer.js
index eb1484bb6..2d359dcb8 100644
--- a/client/players/LocalAudioPlayer.js
+++ b/client/players/LocalAudioPlayer.js
@@ -35,6 +35,8 @@ export default class LocalAudioPlayer extends EventEmitter {
     var audioEl = document.createElement('audio')
     audioEl.id = 'audio-player'
     audioEl.style.display = 'none'
+    audioEl.crossOrigin = 'anonymous'
+
     document.body.appendChild(audioEl)
     this.player = audioEl
 
@@ -135,6 +137,18 @@ export default class LocalAudioPlayer extends EventEmitter {
       this.usingNativeplayer = true
       this.player.src = this.currentTrack.relativeContentUrl
       this.player.currentTime = this.startTime
+
+      if (document.getElementById('transcription-track')) {
+        document.getElementById('transcription-track').remove()
+      }
+      const trackElement = document.createElement("track");
+
+      trackElement.id = "transcription-track";
+      trackElement.kind = "subtitles";
+      trackElement.label = "Transcription";
+      trackElement.default = true;
+      trackElement.src = this.currentTrack.relativeTranscriptionUrl;
+      this.player.appendChild(trackElement);
       return
     }
 
@@ -168,7 +182,7 @@ export default class LocalAudioPlayer extends EventEmitter {
 
     this.hlsInstance.attachMedia(this.player)
     this.hlsInstance.on(Hls.Events.MEDIA_ATTACHED, () => {
-      this.hlsInstance.loadSource(this.currentTrack.relativeContentUrl)
+     this.hlsInstance.loadSource(this.currentTrack.relativeContentUrl)
 
       this.hlsInstance.on(Hls.Events.MANIFEST_PARSED, () => {
         console.log('[HLS] Manifest Parsed')
@@ -206,6 +220,19 @@ export default class LocalAudioPlayer extends EventEmitter {
     this.trackStartTime = Math.max(0, this.startTime - (this.currentTrack.startOffset || 0))
     this.player.src = this.currentTrack.relativeContentUrl
     console.log(`[LocalPlayer] Loading track src ${this.currentTrack.relativeContentUrl}`)
+
+    if (document.getElementById('transcription-track')) {
+      document.getElementById('transcription-track').remove()
+    }
+    const trackElement = document.createElement("track");
+
+    trackElement.id = "transcription-track";
+    trackElement.kind = "subtitles";
+    trackElement.label = "Transcription";
+    trackElement.default = true;
+    trackElement.src = this.currentTrack.relativeTranscriptionUrl;
+    this.player.appendChild(trackElement);
+
     this.player.load()
   }
 
@@ -338,4 +365,4 @@ export default class LocalAudioPlayer extends EventEmitter {
     var last = bufferedRanges[bufferedRanges.length - 1]
     return last.end
   }
-}
\ No newline at end of file
+}
diff --git a/client/strings/en-us.json b/client/strings/en-us.json
index 8fe9d8c7e..873793340 100644
--- a/client/strings/en-us.json
+++ b/client/strings/en-us.json
@@ -562,6 +562,7 @@
   "LabelViewBookmarks": "View bookmarks",
   "LabelViewChapters": "View chapters",
   "LabelViewQueue": "View player queue",
+  "LabelViewTranscription": "View transcription",
   "LabelVolume": "Volume",
   "LabelWeekdaysToRun": "Weekdays to run",
   "LabelYearReviewHide": "Hide Year in Review",
@@ -779,4 +780,4 @@
   "ToastSocketFailedToConnect": "Socket failed to connect",
   "ToastUserDeleteFailed": "Failed to delete user",
   "ToastUserDeleteSuccess": "User deleted"
-}
\ No newline at end of file
+}
diff --git a/server/controllers/LibraryItemController.js b/server/controllers/LibraryItemController.js
index e987dda05..f704e9b04 100644
--- a/server/controllers/LibraryItemController.js
+++ b/server/controllers/LibraryItemController.js
@@ -22,9 +22,9 @@ class LibraryItemController {
    * Optional query params:
    * ?include=progress,rssfeed,downloads
    * ?expanded=1
-   * 
-   * @param {import('express').Request} req 
-   * @param {import('express').Response} res 
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
    */
   async findOne(req, res) {
     const includeEntities = (req.query.include || '').split(',')
@@ -88,9 +88,9 @@ class LibraryItemController {
   /**
    * GET: /api/items/:id/download
    * Download library item. Zip file if multiple files.
-   * 
-   * @param {import('express').Request} req 
-   * @param {import('express').Response} res 
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
    */
   download(req, res) {
     if (!req.user.canDownload) {
@@ -120,9 +120,9 @@ class LibraryItemController {
   /**
    * PATCH: /items/:id/media
    * Update media for a library item. Will create new authors & series when necessary
-   * 
-   * @param {import('express').Request} req 
-   * @param {import('express').Response} res 
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
    */
   async updateMedia(req, res) {
     const libraryItem = req.libraryItem
@@ -252,9 +252,9 @@ class LibraryItemController {
 
   /**
    * GET: api/items/:id/cover
-   * 
-   * @param {import('express').Request} req 
-   * @param {import('express').Response} res 
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
    */
   async getCover(req, res) {
     const { query: { width, height, format, raw } } = req
@@ -593,9 +593,9 @@ class LibraryItemController {
   /**
    * GET api/items/:id/ffprobe/:fileid
    * FFProbe JSON result from audio file
-   * 
+   *
    * @param {express.Request} req
-   * @param {express.Response} res 
+   * @param {express.Response} res
    */
   async getFFprobeData(req, res) {
     if (!req.user.isAdminOrUp) {
@@ -619,9 +619,9 @@ class LibraryItemController {
 
   /**
    * GET api/items/:id/file/:fileid
-   * 
+   *
    * @param {express.Request} req
-   * @param {express.Response} res 
+   * @param {express.Response} res
    */
   async getLibraryFile(req, res) {
     const libraryFile = req.libraryFile
@@ -641,10 +641,35 @@ class LibraryItemController {
   }
 
   /**
-   * DELETE api/items/:id/file/:fileid
-   * 
+   * GET api/items/:id/file/:fileid/transcript
+   *
    * @param {express.Request} req
-   * @param {express.Response} res 
+   * @param {express.Response} res
+   */
+  async getTranscriptionFile(req, res) {
+    const libraryFile = req.libraryFile
+
+    const baseName = Path.basename(libraryFile.metadata.path, Path.extname(libraryFile.metadata.path));
+
+    const vttFilePath = Path.join(Path.dirname(libraryFile.metadata.path), `${baseName}.vtt`);
+
+    if (global.XAccel) {
+      const encodedURI = encodeUriPath(global.XAccel + vttFilePath)
+      Logger.debug(`Use X-Accel to serve static file ${encodedURI}`)
+      return res.status(204).header({ 'X-Accel-Redirect': encodedURI }).send()
+    }
+
+    // Set the correct mimetype for .vtt files
+    res.setHeader('Content-Type', 'text/vtt');
+
+    res.sendFile(vttFilePath);
+  }
+
+  /**
+   * DELETE api/items/:id/file/:fileid
+   *
+   * @param {express.Request} req
+   * @param {express.Response} res
    */
   async deleteLibraryFile(req, res) {
     const libraryFile = req.libraryFile
@@ -672,7 +697,7 @@ class LibraryItemController {
    * GET api/items/:id/file/:fileid/download
    * Same as GET api/items/:id/file/:fileid but allows logging and restricting downloads
    * @param {express.Request} req
-   * @param {express.Response} res 
+   * @param {express.Response} res
    */
   async downloadLibraryFile(req, res) {
     const libraryFile = req.libraryFile
@@ -704,9 +729,9 @@ class LibraryItemController {
    * fileid is the inode value stored in LibraryFile.ino or EBookFile.ino
    * fileid is only required when reading a supplementary ebook
    * when no fileid is passed in the primary ebook will be returned
-   * 
+   *
    * @param {express.Request} req
-   * @param {express.Response} res 
+   * @param {express.Response} res
    */
   async getEBookFile(req, res) {
     let ebookFile = null
@@ -740,9 +765,9 @@ class LibraryItemController {
    * toggle the status of an ebook file.
    * if an ebook file is the primary ebook, then it will be changed to supplementary
    * if an ebook file is supplementary, then it will be changed to primary
-   * 
+   *
    * @param {express.Request} req
-   * @param {express.Response} res 
+   * @param {express.Response} res
    */
   async updateEbookFileStatus(req, res) {
     const ebookLibraryFile = req.libraryItem.libraryFiles.find(lf => lf.ino === req.params.fileid)
@@ -797,4 +822,4 @@ class LibraryItemController {
     next()
   }
 }
-module.exports = new LibraryItemController()
\ No newline at end of file
+module.exports = new LibraryItemController()
diff --git a/server/objects/files/AudioTrack.js b/server/objects/files/AudioTrack.js
index b50d3e3f8..6776098ce 100644
--- a/server/objects/files/AudioTrack.js
+++ b/server/objects/files/AudioTrack.js
@@ -5,6 +5,7 @@ class AudioTrack {
     this.duration = null
     this.title = null
     this.contentUrl = null
+    this.transcriptUrl = null
     this.mimeType = null
     this.codec = null
     this.metadata = null
@@ -17,6 +18,7 @@ class AudioTrack {
       duration: this.duration,
       title: this.title,
       contentUrl: this.contentUrl,
+      transcriptUrl: this.transcriptUrl,
       mimeType: this.mimeType,
       codec: this.codec,
       metadata: this.metadata?.toJSON() || null
@@ -30,6 +32,7 @@ class AudioTrack {
     this.title = audioFile.metadata.filename || ''
 
     this.contentUrl = `${global.RouterBasePath}/api/items/${itemId}/file/${audioFile.ino}`
+    this.transcriptUrl = `${global.RouterBasePath}/api/items/${itemId}/file/${audioFile.ino}/transcript`
     this.mimeType = audioFile.mimeType
     this.codec = audioFile.codec || null
     this.metadata = audioFile.metadata.clone()
@@ -44,4 +47,4 @@ class AudioTrack {
     this.mimeType = 'application/vnd.apple.mpegurl'
   }
 }
-module.exports = AudioTrack
\ No newline at end of file
+module.exports = AudioTrack
diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js
index 43e7c907e..bf2a52b55 100644
--- a/server/routers/ApiRouter.js
+++ b/server/routers/ApiRouter.js
@@ -116,6 +116,7 @@ class ApiRouter {
     this.router.post('/items/:id/chapters', LibraryItemController.middleware.bind(this), LibraryItemController.updateMediaChapters.bind(this))
     this.router.get('/items/:id/ffprobe/:fileid', LibraryItemController.middleware.bind(this), LibraryItemController.getFFprobeData.bind(this))
     this.router.get('/items/:id/file/:fileid', LibraryItemController.middleware.bind(this), LibraryItemController.getLibraryFile.bind(this))
+    this.router.get('/items/:id/file/:fileid/transcript', LibraryItemController.middleware.bind(this), LibraryItemController.getTranscriptionFile.bind(this))
     this.router.delete('/items/:id/file/:fileid', LibraryItemController.middleware.bind(this), LibraryItemController.deleteLibraryFile.bind(this))
     this.router.get('/items/:id/file/:fileid/download', LibraryItemController.middleware.bind(this), LibraryItemController.downloadLibraryFile.bind(this))
     this.router.get('/items/:id/ebook/:fileid?', LibraryItemController.middleware.bind(this), LibraryItemController.getEBookFile.bind(this))
@@ -425,9 +426,9 @@ class ApiRouter {
   /**
    * Used when a series is removed from a book
    * Series is removed if it only has 1 book
-   * 
+   *
    * @param {string} bookId
-   * @param {string[]} seriesIds 
+   * @param {string[]} seriesIds
    */
   async checkRemoveEmptySeries(bookId, seriesIds) {
     if (!seriesIds?.length) return
@@ -455,7 +456,7 @@ class ApiRouter {
 
   /**
    * Remove an empty series & close an open RSS feed
-   * @param {import('../models/Series')} series 
+   * @param {import('../models/Series')} series
    */
   async removeEmptySeries(series) {
     await this.rssFeedManager.closeFeedForEntityId(series.id)

From 8e5fc4a411b0bd42ab9151662d1ff9a254c0433a Mon Sep 17 00:00:00 2001
From: mfcar <marcosfcar@gmail.com>
Date: Sat, 4 May 2024 11:17:41 +0100
Subject: [PATCH 2/9] Avoid duplicated code

---
 client/players/LocalAudioPlayer.js | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/client/players/LocalAudioPlayer.js b/client/players/LocalAudioPlayer.js
index 2d359dcb8..884951c93 100644
--- a/client/players/LocalAudioPlayer.js
+++ b/client/players/LocalAudioPlayer.js
@@ -138,17 +138,7 @@ export default class LocalAudioPlayer extends EventEmitter {
       this.player.src = this.currentTrack.relativeContentUrl
       this.player.currentTime = this.startTime
 
-      if (document.getElementById('transcription-track')) {
-        document.getElementById('transcription-track').remove()
-      }
-      const trackElement = document.createElement("track");
-
-      trackElement.id = "transcription-track";
-      trackElement.kind = "subtitles";
-      trackElement.label = "Transcription";
-      trackElement.default = true;
-      trackElement.src = this.currentTrack.relativeTranscriptionUrl;
-      this.player.appendChild(trackElement);
+      this.createTranscriptionTrack();
       return
     }
 
@@ -221,8 +211,14 @@ export default class LocalAudioPlayer extends EventEmitter {
     this.player.src = this.currentTrack.relativeContentUrl
     console.log(`[LocalPlayer] Loading track src ${this.currentTrack.relativeContentUrl}`)
 
+    this.createTranscriptionTrack();
+
+    this.player.load()
+  }
+
+  createTranscriptionTrack() {
     if (document.getElementById('transcription-track')) {
-      document.getElementById('transcription-track').remove()
+      document.getElementById('transcription-track').remove();
     }
     const trackElement = document.createElement("track");
 
@@ -232,8 +228,6 @@ export default class LocalAudioPlayer extends EventEmitter {
     trackElement.default = true;
     trackElement.src = this.currentTrack.relativeTranscriptionUrl;
     this.player.appendChild(trackElement);
-
-    this.player.load()
   }
 
   destroyHlsInstance() {

From ee8e7cf958524b2849239e1a236fe737587378cc Mon Sep 17 00:00:00 2001
From: mfcar <marcosfcar@gmail.com>
Date: Sat, 4 May 2024 12:21:07 +0100
Subject: [PATCH 3/9] Add seek support to transcriptions

---
 client/components/app/MediaPlayerContainer.vue | 9 ++++++---
 client/components/player/TranscriptionLine.vue | 8 +++++++-
 client/components/player/TranscriptionUi.vue   | 4 ++++
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/client/components/app/MediaPlayerContainer.vue b/client/components/app/MediaPlayerContainer.vue
index c34e8595b..f384c7dbf 100644
--- a/client/components/app/MediaPlayerContainer.vue
+++ b/client/components/app/MediaPlayerContainer.vue
@@ -56,7 +56,10 @@
       @showTranscription="showTranscriptionUi = !showTranscriptionUi"
     />
 
-    <transcription-ui v-if="showTranscriptionUi"></transcription-ui>
+    <transcription-ui
+      v-if="showTranscriptionUi"
+      @seek="seek"
+    />
 
     <modals-bookmarks-modal v-model="showBookmarksModal" :bookmarks="bookmarks" :current-time="bookmarkCurrentTime" :library-item-id="libraryItemId" @select="selectBookmark" />
 
@@ -95,8 +98,8 @@ export default {
     }
   },
   watch: {
-    'playerHandler.playerState': function (newVal, oldVal) {
-      // Refresh the transcription UI when the player is changed
+    'playerHandler.playerState': function (newVal) {
+      // Refresh the transcription UI when the audio track is changed
       if (newVal === 'LOADED') {
         this.showTranscriptionUi = false;
         this.$nextTick(() => {
diff --git a/client/components/player/TranscriptionLine.vue b/client/components/player/TranscriptionLine.vue
index ff074977e..9f5ac3ddd 100644
--- a/client/components/player/TranscriptionLine.vue
+++ b/client/components/player/TranscriptionLine.vue
@@ -1,5 +1,5 @@
 <template>
-  <div :class="{ 'text-warning': isActive }">
+  <div :class="{ 'text-warning': isActive }"  class="cursor-pointer" @click.stop="clickSeek">
     <div v-html="cue.text"></div>
   </div>
 </template>
@@ -14,6 +14,12 @@ export default {
       isActive: false
     };
   },
+  methods: {
+    clickSeek() {
+      const time = this.cue.startTime;
+      this.$emit('seek', time);
+    }
+  },
   created() {
     this.cue.onenter = () => (this.isActive = true);
     this.cue.onexit = () => (this.isActive = false);
diff --git a/client/components/player/TranscriptionUi.vue b/client/components/player/TranscriptionUi.vue
index 73d807f2b..11f013b3c 100644
--- a/client/components/player/TranscriptionUi.vue
+++ b/client/components/player/TranscriptionUi.vue
@@ -5,6 +5,7 @@
       :key="index"
       :cue="cue"
       ref="transcriptionLine + index"
+      @seek="seek"
     ></transcription-line>
   </div>
 </template>
@@ -29,6 +30,9 @@ export default {
       const trackElement = document.getElementById("transcription-track");
       this.cues = trackElement.track.cues;
     },
+    seek(time) {
+      this.$emit('seek', time)
+    },
   },
 };
 </script>

From 282203a30aa16091c2fe15301182a01cb42ed0ef Mon Sep 17 00:00:00 2001
From: mfcar <marcosfcar@gmail.com>
Date: Sat, 4 May 2024 12:52:04 +0100
Subject: [PATCH 4/9] Automatically scrolls to active cue when enable/disable
 the transcription panel

---
 client/components/player/TranscriptionLine.vue | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/client/components/player/TranscriptionLine.vue b/client/components/player/TranscriptionLine.vue
index 9f5ac3ddd..219461056 100644
--- a/client/components/player/TranscriptionLine.vue
+++ b/client/components/player/TranscriptionLine.vue
@@ -16,18 +16,26 @@ export default {
   },
   methods: {
     clickSeek() {
-      const time = this.cue.startTime;
-      this.$emit('seek', time);
+      const time = this.cue.startTime
+      this.$emit('seek', time)
+    },
+    scrollIntoView() {
+      this.$el.scrollIntoView({behavior: 'smooth'})
+    }
+  },
+  mounted() {
+    if (this.isActive) {
+      this.scrollIntoView()
     }
   },
   created() {
-    this.cue.onenter = () => (this.isActive = true);
-    this.cue.onexit = () => (this.isActive = false);
+    this.cue.onenter = () => (this.isActive = true)
+    this.cue.onexit = () => (this.isActive = false)
   },
   watch: {
     isActive(newVal) {
       if (newVal) {
-        this.$el.scrollIntoView({behavior: 'smooth'});
+        this.scrollIntoView()
       }
     }
   }

From 68d4dac4a20308e0cc679f6fe8b9475d9680ae42 Mon Sep 17 00:00:00 2001
From: mfcar <marcosfcar@gmail.com>
Date: Sat, 4 May 2024 14:21:40 +0100
Subject: [PATCH 5/9] Avoid error: "Cannot read properties of null (reading
 'track')"

---
 client/components/app/MediaPlayerContainer.vue |  5 +++--
 client/components/player/TranscriptionUi.vue   | 15 +++++++++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/client/components/app/MediaPlayerContainer.vue b/client/components/app/MediaPlayerContainer.vue
index f384c7dbf..937d6e59b 100644
--- a/client/components/app/MediaPlayerContainer.vue
+++ b/client/components/app/MediaPlayerContainer.vue
@@ -101,9 +101,9 @@ export default {
     'playerHandler.playerState': function (newVal) {
       // Refresh the transcription UI when the audio track is changed
       if (newVal === 'LOADED') {
-        this.showTranscriptionUi = false;
+        this.showTranscriptionUi = false
         this.$nextTick(() => {
-          this.showTranscriptionUi = true;
+          this.showTranscriptionUi = true
         });
       }
     },
@@ -507,6 +507,7 @@ export default {
     this.$eventBus.$on('playback-time-update', this.playbackTimeUpdate)
     this.$eventBus.$on('play-item', this.playLibraryItem)
     this.$eventBus.$on('pause-item', this.pauseItem)
+    this.showTranscriptionUi = false
   },
   beforeDestroy() {
     this.$eventBus.$off('cast-session-active', this.castSessionActive)
diff --git a/client/components/player/TranscriptionUi.vue b/client/components/player/TranscriptionUi.vue
index 11f013b3c..6ea3f9ab1 100644
--- a/client/components/player/TranscriptionUi.vue
+++ b/client/components/player/TranscriptionUi.vue
@@ -17,9 +17,15 @@ export default {
   components: {
     TranscriptionLine
   },
+  watch: {
+    trackElement() {
+      this.setCues();
+    }
+  },
   data() {
     return {
       cues: [],
+      trackElement: null
     };
   },
   mounted() {
@@ -27,12 +33,17 @@ export default {
   },
   methods: {
     init() {
-      const trackElement = document.getElementById("transcription-track");
-      this.cues = trackElement.track.cues;
+      this.trackElement = document.getElementById("transcription-track")
+      if (this.trackElement && this.trackElement.track) {
+        this.setCues()
+      }
     },
     seek(time) {
       this.$emit('seek', time)
     },
+    setCues() {
+      this.cues = this.trackElement.track.cues
+    }
   },
 };
 </script>

From 35f51f4e986eb478a8d649e17bfce97cde631fa8 Mon Sep 17 00:00:00 2001
From: mfcar <marcosfcar@gmail.com>
Date: Sat, 4 May 2024 20:26:05 +0100
Subject: [PATCH 6/9] Fix formatting

---
 client/players/AudioTrack.js       | 69 +++++++++++++++---------------
 client/players/LocalAudioPlayer.js |  2 +-
 2 files changed, 35 insertions(+), 36 deletions(-)

diff --git a/client/players/AudioTrack.js b/client/players/AudioTrack.js
index 2146c3e09..96d6a8d98 100644
--- a/client/players/AudioTrack.js
+++ b/client/players/AudioTrack.js
@@ -1,43 +1,42 @@
 export default class AudioTrack {
-    constructor(track, userToken) {
-        this.index = track.index || 0
-        this.startOffset = track.startOffset || 0 // Total time of all previous tracks
-        this.duration = track.duration || 0
-        this.title = track.title || ''
-        this.contentUrl = track.contentUrl || null
-        this.transcriptUrl = track.transcriptUrl || null
-        this.mimeType = track.mimeType
-        this.metadata = track.metadata || {}
+  constructor(track, userToken) {
+    this.index = track.index || 0
+    this.startOffset = track.startOffset || 0 // Total time of all previous tracks
+    this.duration = track.duration || 0
+    this.title = track.title || ''
+    this.contentUrl = track.contentUrl || null
+    this.mimeType = track.mimeType
+    this.metadata = track.metadata || {}
 
-        this.userToken = userToken
+    this.userToken = userToken
+  }
+
+  get fullContentUrl() {
+    if (!this.contentUrl || this.contentUrl.startsWith('http')) return this.contentUrl
+
+    if (process.env.NODE_ENV === 'development') {
+      return `${process.env.serverUrl}${this.contentUrl}?token=${this.userToken}`
+    }
+    return `${window.location.origin}${this.contentUrl}?token=${this.userToken}`
+  }
+
+  get relativeContentUrl() {
+    if (!this.contentUrl || this.contentUrl.startsWith('http')) return this.contentUrl
+
+    if (process.env.NODE_ENV === 'development') {
+      return `${process.env.serverUrl}${this.contentUrl}?token=${this.userToken}`
     }
 
-    get fullContentUrl() {
-        if (!this.contentUrl || this.contentUrl.startsWith('http')) return this.contentUrl
+    return this.contentUrl + `?token=${this.userToken}`
+  }
 
-        if (process.env.NODE_ENV === 'development') {
-            return `${process.env.serverUrl}${this.contentUrl}?token=${this.userToken}`
-        }
-        return `${window.location.origin}${this.contentUrl}?token=${this.userToken}`
+  get relativeTranscriptionUrl() {
+    if (!this.transcriptUrl || this.transcriptUrl.startsWith('http')) return this.transcriptUrl
+
+    if (process.env.NODE_ENV === 'development') {
+      return `${process.env.serverUrl}${this.transcriptUrl}?token=${this.userToken}`
     }
 
-    get relativeContentUrl() {
-        if (!this.contentUrl || this.contentUrl.startsWith('http')) return this.contentUrl
-
-        if (process.env.NODE_ENV === 'development') {
-            return `${process.env.serverUrl}${this.contentUrl}?token=${this.userToken}`
-        }
-
-        return this.contentUrl + `?token=${this.userToken}`
-    }
-
-    get relativeTranscriptionUrl() {
-        if (!this.transcriptUrl || this.transcriptUrl.startsWith('http')) return this.transcriptUrl
-
-        if (process.env.NODE_ENV === 'development') {
-            return `${process.env.serverUrl}${this.transcriptUrl}?token=${this.userToken}`
-        }
-
-        return this.transcriptUrl + `?token=${this.userToken}`
-    }
+    return this.transcriptUrl + `?token=${this.userToken}`
+  }
 }
diff --git a/client/players/LocalAudioPlayer.js b/client/players/LocalAudioPlayer.js
index 884951c93..f56f89b90 100644
--- a/client/players/LocalAudioPlayer.js
+++ b/client/players/LocalAudioPlayer.js
@@ -172,7 +172,7 @@ export default class LocalAudioPlayer extends EventEmitter {
 
     this.hlsInstance.attachMedia(this.player)
     this.hlsInstance.on(Hls.Events.MEDIA_ATTACHED, () => {
-     this.hlsInstance.loadSource(this.currentTrack.relativeContentUrl)
+    this.hlsInstance.loadSource(this.currentTrack.relativeContentUrl)
 
       this.hlsInstance.on(Hls.Events.MANIFEST_PARSED, () => {
         console.log('[HLS] Manifest Parsed')

From bfcf4e317f40425c60c58b113f890b3df9e60a26 Mon Sep 17 00:00:00 2001
From: mfcar <marcosfcar@gmail.com>
Date: Sat, 4 May 2024 20:31:39 +0100
Subject: [PATCH 7/9] Remove semicolon

---
 .../components/app/MediaPlayerContainer.vue   |  2 +-
 client/components/player/TranscriptionUi.vue  |  8 ++++----
 client/players/LocalAudioPlayer.js            | 20 +++++++++----------
 server/controllers/LibraryItemController.js   |  8 ++++----
 4 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/client/components/app/MediaPlayerContainer.vue b/client/components/app/MediaPlayerContainer.vue
index 937d6e59b..82ef1e8c2 100644
--- a/client/components/app/MediaPlayerContainer.vue
+++ b/client/components/app/MediaPlayerContainer.vue
@@ -71,7 +71,7 @@
 
 <script>
 import PlayerHandler from '@/players/PlayerHandler'
-import TranscriptionUi from "@/components/player/TranscriptionUi.vue";
+import TranscriptionUi from "@/components/player/TranscriptionUi.vue"
 
 export default {
   components: {TranscriptionUi},
diff --git a/client/components/player/TranscriptionUi.vue b/client/components/player/TranscriptionUi.vue
index 6ea3f9ab1..773463ce5 100644
--- a/client/components/player/TranscriptionUi.vue
+++ b/client/components/player/TranscriptionUi.vue
@@ -11,7 +11,7 @@
 </template>
 
 <script>
-import TranscriptionLine from "./TranscriptionLine.vue";
+import TranscriptionLine from "./TranscriptionLine.vue"
 
 export default {
   components: {
@@ -19,14 +19,14 @@ export default {
   },
   watch: {
     trackElement() {
-      this.setCues();
+      this.setCues()
     }
   },
   data() {
     return {
       cues: [],
       trackElement: null
-    };
+    }
   },
   mounted() {
     this.init()
@@ -45,7 +45,7 @@ export default {
       this.cues = this.trackElement.track.cues
     }
   },
-};
+}
 </script>
 <style>
 #transcription-panel {
diff --git a/client/players/LocalAudioPlayer.js b/client/players/LocalAudioPlayer.js
index f56f89b90..e8392c177 100644
--- a/client/players/LocalAudioPlayer.js
+++ b/client/players/LocalAudioPlayer.js
@@ -138,7 +138,7 @@ export default class LocalAudioPlayer extends EventEmitter {
       this.player.src = this.currentTrack.relativeContentUrl
       this.player.currentTime = this.startTime
 
-      this.createTranscriptionTrack();
+      this.createTranscriptionTrack()
       return
     }
 
@@ -211,23 +211,23 @@ export default class LocalAudioPlayer extends EventEmitter {
     this.player.src = this.currentTrack.relativeContentUrl
     console.log(`[LocalPlayer] Loading track src ${this.currentTrack.relativeContentUrl}`)
 
-    this.createTranscriptionTrack();
+    this.createTranscriptionTrack()
 
     this.player.load()
   }
 
   createTranscriptionTrack() {
     if (document.getElementById('transcription-track')) {
-      document.getElementById('transcription-track').remove();
+      document.getElementById('transcription-track').remove()
     }
-    const trackElement = document.createElement("track");
+    const trackElement = document.createElement("track")
 
-    trackElement.id = "transcription-track";
-    trackElement.kind = "subtitles";
-    trackElement.label = "Transcription";
-    trackElement.default = true;
-    trackElement.src = this.currentTrack.relativeTranscriptionUrl;
-    this.player.appendChild(trackElement);
+    trackElement.id = "transcription-track"
+    trackElement.kind = "subtitles"
+    trackElement.label = "Transcription"
+    trackElement.default = true
+    trackElement.src = this.currentTrack.relativeTranscriptionUrl
+    this.player.appendChild(trackElement)
   }
 
   destroyHlsInstance() {
diff --git a/server/controllers/LibraryItemController.js b/server/controllers/LibraryItemController.js
index f704e9b04..85c5a6da0 100644
--- a/server/controllers/LibraryItemController.js
+++ b/server/controllers/LibraryItemController.js
@@ -649,9 +649,9 @@ class LibraryItemController {
   async getTranscriptionFile(req, res) {
     const libraryFile = req.libraryFile
 
-    const baseName = Path.basename(libraryFile.metadata.path, Path.extname(libraryFile.metadata.path));
+    const baseName = Path.basename(libraryFile.metadata.path, Path.extname(libraryFile.metadata.path))
 
-    const vttFilePath = Path.join(Path.dirname(libraryFile.metadata.path), `${baseName}.vtt`);
+    const vttFilePath = Path.join(Path.dirname(libraryFile.metadata.path), `${baseName}.vtt`)
 
     if (global.XAccel) {
       const encodedURI = encodeUriPath(global.XAccel + vttFilePath)
@@ -660,9 +660,9 @@ class LibraryItemController {
     }
 
     // Set the correct mimetype for .vtt files
-    res.setHeader('Content-Type', 'text/vtt');
+    res.setHeader('Content-Type', 'text/vtt')
 
-    res.sendFile(vttFilePath);
+    res.sendFile(vttFilePath)
   }
 
   /**

From 1a9aaf17a6f9fd71efad281d38ab501e417ef94e Mon Sep 17 00:00:00 2001
From: mfcar <marcosfcar@gmail.com>
Date: Sat, 4 May 2024 20:49:34 +0100
Subject: [PATCH 8/9] Fix small bug on the AudioTrack

---
 client/players/AudioTrack.js       | 1 +
 client/players/LocalAudioPlayer.js | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/client/players/AudioTrack.js b/client/players/AudioTrack.js
index 96d6a8d98..451f9de01 100644
--- a/client/players/AudioTrack.js
+++ b/client/players/AudioTrack.js
@@ -5,6 +5,7 @@ export default class AudioTrack {
     this.duration = track.duration || 0
     this.title = track.title || ''
     this.contentUrl = track.contentUrl || null
+    this.transcriptUrl = track.transcriptUrl || null
     this.mimeType = track.mimeType
     this.metadata = track.metadata || {}
 
diff --git a/client/players/LocalAudioPlayer.js b/client/players/LocalAudioPlayer.js
index e8392c177..583f546c3 100644
--- a/client/players/LocalAudioPlayer.js
+++ b/client/players/LocalAudioPlayer.js
@@ -220,6 +220,7 @@ export default class LocalAudioPlayer extends EventEmitter {
     if (document.getElementById('transcription-track')) {
       document.getElementById('transcription-track').remove()
     }
+
     const trackElement = document.createElement("track")
 
     trackElement.id = "transcription-track"
@@ -227,6 +228,7 @@ export default class LocalAudioPlayer extends EventEmitter {
     trackElement.label = "Transcription"
     trackElement.default = true
     trackElement.src = this.currentTrack.relativeTranscriptionUrl
+
     this.player.appendChild(trackElement)
   }
 

From 2f515cc2075599c6b2589948b332ceadb2c50400 Mon Sep 17 00:00:00 2001
From: mfcar <marcosfcar@gmail.com>
Date: Sat, 4 May 2024 20:50:18 +0100
Subject: [PATCH 9/9] Add support to recognize srt and vtt as subtitles formats
 on the file table

---
 client/plugins/constants.js         | 5 +++--
 server/objects/files/LibraryFile.js | 3 ++-
 server/utils/globals.js             | 3 ++-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/client/plugins/constants.js b/client/plugins/constants.js
index f001f6ced..8b95887f1 100644
--- a/client/plugins/constants.js
+++ b/client/plugins/constants.js
@@ -4,7 +4,8 @@ const SupportedFileTypes = {
   ebook: ['epub', 'pdf', 'mobi', 'azw3', 'cbr', 'cbz'],
   info: ['nfo'],
   text: ['txt'],
-  metadata: ['opf', 'abs', 'xml', 'json']
+  metadata: ['opf', 'abs', 'xml', 'json'],
+  subtitle: ['srt', 'vtt']
 }
 
 const DownloadStatus = {
@@ -82,4 +83,4 @@ export default ({ app }, inject) => {
   inject('constants', Constants)
   inject('keynames', KeyNames)
   inject('hotkeys', Hotkeys)
-}
\ No newline at end of file
+}
diff --git a/server/objects/files/LibraryFile.js b/server/objects/files/LibraryFile.js
index 395e11cc1..e024fb4e4 100644
--- a/server/objects/files/LibraryFile.js
+++ b/server/objects/files/LibraryFile.js
@@ -46,6 +46,7 @@ class LibraryFile {
     if (globals.SupportedVideoTypes.includes(this.metadata.format)) return 'video'
     if (globals.TextFileTypes.includes(this.metadata.format)) return 'text'
     if (globals.MetadataFileTypes.includes(this.metadata.format)) return 'metadata'
+    if (globals.SubtitleFileTypes.includes(this.metadata.format)) return 'subtitle'
     return 'unknown'
   }
 
@@ -75,4 +76,4 @@ class LibraryFile {
     this.updatedAt = Date.now()
   }
 }
-module.exports = LibraryFile
\ No newline at end of file
+module.exports = LibraryFile
diff --git a/server/utils/globals.js b/server/utils/globals.js
index b24fc76d0..04c9bbe61 100644
--- a/server/utils/globals.js
+++ b/server/utils/globals.js
@@ -4,7 +4,8 @@ const globals = {
   SupportedEbookTypes: ['epub', 'pdf', 'mobi', 'azw3', 'cbr', 'cbz'],
   SupportedVideoTypes: ['mp4'],
   TextFileTypes: ['txt', 'nfo'],
-  MetadataFileTypes: ['opf', 'abs', 'xml', 'json']
+  MetadataFileTypes: ['opf', 'abs', 'xml', 'json'],
+  SubtitleFileTypes: ['srt', 'vtt']
 }
 
 module.exports = globals