Merge e1ceac63e2 into 1bad2d9072

.github/workflows/build-and-publish.yaml

@@ -0,0 +1,73 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches:
+      - '5108-chunked-upload-support'
+  pull_request:
+    branches:
+      - '5108-chunked-upload-support'
+
+env:
+  REGISTRY: ghcr.io
+  # This automatically resolves to kc9wwh/audiobookshelf
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  test-and-build:
+    name: Test Node Build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20' # Standard for modern ABS
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run tests
+        run: npm test --if-present
+
+  docker-publish:
+    name: Build and Push to GHCR
+    needs: test-and-build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write # Critical: allows pushing the image to ghcr.io
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=sha
+            # This creates a tag like ghcr.io/kc9wwh/audiobookshelf:5108-chunked-upload-support
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

client/pages/upload/index.vue

@@ -307,40 +307,101 @@ export default {
       }
     },
     async uploadItem(item) {
-      var form = new FormData()
-      form.set('title', item.title)
-      if (!this.selectedLibraryIsPodcast) {
-        form.set('author', item.author || '')
-        form.set('series', item.series || '')
-      }
-      form.set('library', this.selectedLibraryId)
-      form.set('folder', this.selectedFolderId)
+      const CHUNK_SIZE = 50 * 1024 * 1024 // 50MB
+      let allFilesSuccessful = true
 
-      var index = 0
-      item.files.forEach((file) => {
-        form.set(`${index++}`, file)
-      })
+      for (let fileIndex = 0; fileIndex < item.files.length; fileIndex++) {
+        const file = item.files[fileIndex]
 
-      const config = {
-        onUploadProgress: (progressEvent) => {
-          if (progressEvent.lengthComputable) {
-            const progress = {
-              loaded: progressEvent.loaded,
-              total: progressEvent.total
+        if (file.size <= CHUNK_SIZE) {
+          // Legacy upload for small files
+          var form = new FormData()
+          form.set('title', item.title)
+          if (!this.selectedLibraryIsPodcast) {
+            form.set('author', item.author || '')
+            form.set('series', item.series || '')
+          }
+          form.set('library', this.selectedLibraryId)
+          form.set('folder', this.selectedFolderId)
+          form.set(`${fileIndex}`, file)
+
+          const config = {
+            onUploadProgress: (progressEvent) => {
+              if (progressEvent.lengthComputable) {
+                this.updateItemCardProgress(item.index, { loaded: progressEvent.loaded, total: progressEvent.total })
+              }
+            }
+          }
+
+          const success = await this.$axios
+            .$post('/api/upload', form, config)
+            .then(() => true)
+            .catch((error) => {
+              console.error('Failed to upload item', error)
+              this.$toast.error(error.response?.data || 'Oops, something went wrong...')
+              return false
+            })
+          if (!success) allFilesSuccessful = false
+        } else {
+          // Chunked upload
+          const fileId = crypto.randomUUID ? crypto.randomUUID() : Math.random().toString(36).substring(2) + Date.now().toString(36)
+          const totalChunks = Math.ceil(file.size / CHUNK_SIZE)
+
+          for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
+            const start = chunkIndex * CHUNK_SIZE
+            const end = Math.min(start + CHUNK_SIZE, file.size)
+            const chunk = file.slice(start, end)
+
+            var chunkForm = new FormData()
+            chunkForm.set('fileId', fileId)
+            chunkForm.set('chunkIndex', chunkIndex)
+            chunkForm.set('totalChunks', totalChunks)
+            chunkForm.set('filename', file.name)
+            chunkForm.set('title', item.title)
+            if (!this.selectedLibraryIsPodcast) {
+              chunkForm.set('author', item.author || '')
+              chunkForm.set('series', item.series || '')
+            }
+            chunkForm.set('library', this.selectedLibraryId)
+            chunkForm.set('folder', this.selectedFolderId)
+            chunkForm.set('chunk', chunk)
+
+            let retryCount = 0
+            const maxRetries = 1
+            let chunkSuccess = false
+
+            while (retryCount <= maxRetries && !chunkSuccess) {
+              try {
+                const config = {
+                  onUploadProgress: (progressEvent) => {
+                    if (progressEvent.lengthComputable) {
+                      // Calculate overall progress for this specific file
+                      const totalLoaded = chunkIndex * CHUNK_SIZE + progressEvent.loaded
+                      this.updateItemCardProgress(item.index, { loaded: totalLoaded, total: file.size })
+                    }
+                  }
+                }
+
+                await this.$axios.$post('/api/upload/chunk', chunkForm, config)
+                chunkSuccess = true
+              } catch (error) {
+                retryCount++
+                console.error(`Failed to upload chunk ${chunkIndex} for ${file.name} (Attempt ${retryCount})`, error)
+                if (retryCount > maxRetries) {
+                  this.$toast.error(`Failed to upload part of ${file.name}. Upload aborted.`)
+                  allFilesSuccessful = false
+                  break
+                }
+              }
+            }
+
+            if (!chunkSuccess) {
+              break // Break the outer chunk loop if retries exhausted
             }
-            this.updateItemCardProgress(item.index, progress)
           }
         }
       }
-
-      return this.$axios
-        .$post('/api/upload', form, config)
-        .then(() => true)
-        .catch((error) => {
-          console.error('Failed to upload item', error)
-          this.$toast.error(error.response?.data || 'Oops, something went wrong...')
-          return false
-        })
+      return allFilesSuccessful
     },
     validateItems() {
       var itemData = []

index.js

@@ -1,3 +1,9 @@
+// Node 25 compatibility: SlowBuffer is removed
+const buffer = require('buffer')
+if (!buffer.SlowBuffer) {
+  buffer.SlowBuffer = buffer.Buffer
+}
+
 const optionDefinitions = [
   { name: 'config', alias: 'c', type: String },
   { name: 'metadata', alias: 'm', type: String },

package-lock.json

@@ -2113,6 +2113,21 @@
       "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
       "devOptional": true
     },
+    "node_modules/fsevents": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
+      "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/function-bind": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",

server/controllers/MiscController.js

@@ -99,6 +99,91 @@ class MiscController {
     res.sendStatus(200)
   }
 
+  /**
+   * POST: /api/upload/chunk
+   * Handle chunked upload
+   *
+   * @param {RequestWithUser} req
+   * @param {Response} res
+   */
+  async handleChunkUpload(req, res) {
+    if (!req.user.canUpload) {
+      Logger.warn(`User "${req.user.username}" attempted to upload without permission`)
+      return res.sendStatus(403)
+    }
+    if (!req.files || !Object.values(req.files).length) {
+      Logger.error('Invalid request, no files')
+      return res.sendStatus(400)
+    }
+
+    const { fileId, chunkIndex, totalChunks, filename, library: libraryId, folder: folderId, title, author, series } = req.body
+
+    if (!fileId || chunkIndex === undefined || !totalChunks || !filename || !libraryId || !folderId || !title) {
+      return res.status(400).send('Invalid request body for chunk upload')
+    }
+
+    const library = await Database.libraryModel.findByIdWithFolders(libraryId)
+    if (!library) {
+      return res.status(404).send('Library not found')
+    }
+    if (!req.user.checkCanAccessLibrary(library.id)) {
+      Logger.error(`[MiscController] User "${req.user.username}" attempting to upload to library "${library.id}" without access`)
+      return res.sendStatus(403)
+    }
+
+    const tmpDir = Path.join(global.MetadataPath, 'tmp', 'uploads', fileId)
+    await fs.ensureDir(tmpDir)
+
+    const file = Object.values(req.files)[0]
+    const chunkPath = Path.join(tmpDir, `${fileId}_${chunkIndex}`)
+
+    try {
+      await file.mv(chunkPath)
+
+      // Reassembly logic
+      if (parseInt(chunkIndex) === parseInt(totalChunks) - 1) {
+        const folder = library.libraryFolders.find((fold) => fold.id === folderId)
+        if (!folder) return res.status(404).send('Folder not found')
+
+        const outputDirectoryParts = library.isPodcast ? [title] : [author, series, title]
+        const cleanedOutputDirectoryParts = outputDirectoryParts.filter(Boolean).map((part) => sanitizeFilename(part))
+        const outputDirectory = Path.join(...[folder.path, ...cleanedOutputDirectoryParts])
+        await fs.ensureDir(outputDirectory)
+
+        const finalFilePath = Path.join(outputDirectory, sanitizeFilename(filename))
+        const writeStream = fs.createWriteStream(finalFilePath)
+
+        for (let i = 0; i < totalChunks; i++) {
+          const currentChunkPath = Path.join(tmpDir, `${fileId}_${i}`)
+          const data = await fs.readFile(currentChunkPath)
+          if (!writeStream.write(data)) {
+            await new Promise((resolve) => writeStream.once('drain', resolve))
+          }
+        }
+
+        writeStream.end()
+
+        writeStream.on('finish', async () => {
+          Logger.info(`Successfully merged ${totalChunks} chunks for file ${filename}`)
+          await fs.remove(tmpDir) // Cleanup
+          res.sendStatus(200)
+        })
+
+        writeStream.on('error', async (error) => {
+          Logger.error(`Error merging chunks for file ${filename}`, error)
+          await fs.remove(tmpDir).catch((e) => Logger.error('Failed to clean up temp dir on merge error', e)) // Cleanup on error
+          res.status(500).send('Error merging file')
+        })
+      } else {
+        res.sendStatus(200) // Chunk saved, waiting for more
+      }
+    } catch (error) {
+      Logger.error(`Failed to move chunk ${chunkIndex} for file ${fileId}`, error)
+      await fs.remove(tmpDir).catch((e) => Logger.error('Failed to clean up temp dir on chunk error', e)) // Cleanup
+      return res.status(500).send('Failed to save chunk')
+    }
+  }
+
   /**
    * GET: /api/tasks
    * Get tasks for task manager

server/libs/jwa/buffer-equal-constant-time/index.js

@@ -28,14 +28,19 @@ function bufferEq(a, b) {
 }
 
 bufferEq.install = function () {
-  Buffer.prototype.equal = SlowBuffer.prototype.equal = function equal(that) {
+  Buffer.prototype.equal = function equal(that) {
     return bufferEq(this, that);
   };
+  if (SlowBuffer && SlowBuffer.prototype) {
+    SlowBuffer.prototype.equal = Buffer.prototype.equal;
+  }
 };
 
 var origBufEqual = Buffer.prototype.equal;
-var origSlowBufEqual = SlowBuffer.prototype.equal;
+var origSlowBufEqual = SlowBuffer && SlowBuffer.prototype ? SlowBuffer.prototype.equal : undefined;
 bufferEq.restore = function () {
   Buffer.prototype.equal = origBufEqual;
-  SlowBuffer.prototype.equal = origSlowBufEqual;
+  if (SlowBuffer && SlowBuffer.prototype) {
+    SlowBuffer.prototype.equal = origSlowBufEqual;
+  }
 };

server/routers/ApiRouter.js

@@ -339,6 +339,7 @@ class ApiRouter {
     // Misc Routes
     //
     this.router.post('/upload', MiscController.handleUpload.bind(this))
+    this.router.post('/upload/chunk', MiscController.handleChunkUpload.bind(this))
     this.router.get('/tasks', MiscController.getTasks.bind(this))
     this.router.patch('/settings', MiscController.updateServerSettings.bind(this))
     this.router.patch('/sorting-prefixes', MiscController.updateSortingPrefixes.bind(this))