From 0fb87e1c96ccd25f6a00a314bb3b7bfcebde93db Mon Sep 17 00:00:00 2001 From: Josh Roskos Date: Sat, 16 May 2026 21:10:11 -0500 Subject: [PATCH 1/7] Commit 1: Backend Route & Temporary Storage --- server/controllers/MiscController.js | 51 ++++++++++++++++++++++++++++ server/routers/ApiRouter.js | 1 + 2 files changed, 52 insertions(+) diff --git a/server/controllers/MiscController.js b/server/controllers/MiscController.js index 591f8ccf2..26a3d9f61 100644 --- a/server/controllers/MiscController.js +++ b/server/controllers/MiscController.js @@ -99,6 +99,57 @@ class MiscController { res.sendStatus(200) } + /** + * POST: /api/upload/chunk + * Handle chunked upload + * + * @param {RequestWithUser} req + * @param {Response} res + */ + async handleChunkUpload(req, res) { + if (!req.user.canUpload) { + Logger.warn(`User "${req.user.username}" attempted to upload without permission`) + return res.sendStatus(403) + } + if (!req.files || !Object.values(req.files).length) { + Logger.error('Invalid request, no files') + return res.sendStatus(400) + } + + const { fileId, chunkIndex, totalChunks, filename, library: libraryId, folder: folderId, title, author, series } = req.body + + if (!fileId || chunkIndex === undefined || !totalChunks || !filename || !libraryId || !folderId || !title) { + return res.status(400).send('Invalid request body for chunk upload') + } + + const library = await Database.libraryModel.findByIdWithFolders(libraryId) + if (!library) { + return res.status(404).send('Library not found') + } + if (!req.user.checkCanAccessLibrary(library.id)) { + Logger.error(`[MiscController] User "${req.user.username}" attempting to upload to library "${library.id}" without access`) + return res.sendStatus(403) + } + + const tmpDir = Path.join(global.MetadataPath, 'tmp', 'uploads', fileId) + await fs.ensureDir(tmpDir) + + const file = Object.values(req.files)[0] + const chunkPath = Path.join(tmpDir, `${fileId}_${chunkIndex}`) + + try { + await file.mv(chunkPath) + + // If this is the last chunk, we trigger reassembly. + // To keep the commit scoped to "Temporary Storage", we will return success here for now + // and implement the reassembly in Commit 2. + res.sendStatus(200) + } catch (error) { + Logger.error(`Failed to move chunk ${chunkIndex} for file ${fileId}`, error) + return res.status(500).send('Failed to save chunk') + } + } + /** * GET: /api/tasks * Get tasks for task manager diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index e89b364f3..95de5c957 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -339,6 +339,7 @@ class ApiRouter { // Misc Routes // this.router.post('/upload', MiscController.handleUpload.bind(this)) + this.router.post('/upload/chunk', MiscController.handleChunkUpload.bind(this)) this.router.get('/tasks', MiscController.getTasks.bind(this)) this.router.patch('/settings', MiscController.updateServerSettings.bind(this)) this.router.patch('/sorting-prefixes', MiscController.updateSortingPrefixes.bind(this)) From 7bc04e5e16341fd3e00f0083dae392aa84934bef Mon Sep 17 00:00:00 2001 From: Josh Roskos Date: Sat, 16 May 2026 21:10:20 -0500 Subject: [PATCH 2/7] Commit 2: Backend File Reassembly & Cleanup --- server/controllers/MiscController.js | 42 +++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/server/controllers/MiscController.js b/server/controllers/MiscController.js index 26a3d9f61..cd2d95702 100644 --- a/server/controllers/MiscController.js +++ b/server/controllers/MiscController.js @@ -140,12 +140,46 @@ class MiscController { try { await file.mv(chunkPath) - // If this is the last chunk, we trigger reassembly. - // To keep the commit scoped to "Temporary Storage", we will return success here for now - // and implement the reassembly in Commit 2. - res.sendStatus(200) + // Reassembly logic + if (parseInt(chunkIndex) === parseInt(totalChunks) - 1) { + const folder = library.libraryFolders.find((fold) => fold.id === folderId) + if (!folder) return res.status(404).send('Folder not found') + + const outputDirectoryParts = library.isPodcast ? [title] : [author, series, title] + const cleanedOutputDirectoryParts = outputDirectoryParts.filter(Boolean).map((part) => sanitizeFilename(part)) + const outputDirectory = Path.join(...[folder.path, ...cleanedOutputDirectoryParts]) + await fs.ensureDir(outputDirectory) + + const finalFilePath = Path.join(outputDirectory, sanitizeFilename(filename)) + const writeStream = fs.createWriteStream(finalFilePath) + + for (let i = 0; i < totalChunks; i++) { + const currentChunkPath = Path.join(tmpDir, `${fileId}_${i}`) + const data = await fs.readFile(currentChunkPath) + if (!writeStream.write(data)) { + await new Promise((resolve) => writeStream.once('drain', resolve)) + } + } + + writeStream.end() + + writeStream.on('finish', async () => { + Logger.info(`Successfully merged ${totalChunks} chunks for file ${filename}`) + await fs.remove(tmpDir) // Cleanup + res.sendStatus(200) + }) + + writeStream.on('error', async (error) => { + Logger.error(`Error merging chunks for file ${filename}`, error) + await fs.remove(tmpDir).catch((e) => Logger.error('Failed to clean up temp dir on merge error', e)) // Cleanup on error + res.status(500).send('Error merging file') + }) + } else { + res.sendStatus(200) // Chunk saved, waiting for more + } } catch (error) { Logger.error(`Failed to move chunk ${chunkIndex} for file ${fileId}`, error) + await fs.remove(tmpDir).catch((e) => Logger.error('Failed to clean up temp dir on chunk error', e)) // Cleanup return res.status(500).send('Failed to save chunk') } } From 2ab36b42df54939eb4093b57a91f8b4a17bc753e Mon Sep 17 00:00:00 2001 From: Josh Roskos Date: Sat, 16 May 2026 21:10:31 -0500 Subject: [PATCH 3/7] Commit 3: Frontend Chunking Logic --- client/pages/upload/index.vue | 93 ++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 28 deletions(-) diff --git a/client/pages/upload/index.vue b/client/pages/upload/index.vue index adc21ff90..e4655dc75 100644 --- a/client/pages/upload/index.vue +++ b/client/pages/upload/index.vue @@ -307,40 +307,77 @@ export default { } }, async uploadItem(item) { - var form = new FormData() - form.set('title', item.title) - if (!this.selectedLibraryIsPodcast) { - form.set('author', item.author || '') - form.set('series', item.series || '') - } - form.set('library', this.selectedLibraryId) - form.set('folder', this.selectedFolderId) + const CHUNK_SIZE = 50 * 1024 * 1024 // 50MB + let allFilesSuccessful = true - var index = 0 - item.files.forEach((file) => { - form.set(`${index++}`, file) - }) + for (let fileIndex = 0; fileIndex < item.files.length; fileIndex++) { + const file = item.files[fileIndex] - const config = { - onUploadProgress: (progressEvent) => { - if (progressEvent.lengthComputable) { - const progress = { - loaded: progressEvent.loaded, - total: progressEvent.total + if (file.size <= CHUNK_SIZE) { + // Legacy upload for small files + var form = new FormData() + form.set('title', item.title) + if (!this.selectedLibraryIsPodcast) { + form.set('author', item.author || '') + form.set('series', item.series || '') + } + form.set('library', this.selectedLibraryId) + form.set('folder', this.selectedFolderId) + form.set(`${fileIndex}`, file) + + const config = { + onUploadProgress: (progressEvent) => { + if (progressEvent.lengthComputable) { + this.updateItemCardProgress(item.index, { loaded: progressEvent.loaded, total: progressEvent.total }) + } + } + } + + const success = await this.$axios + .$post('/api/upload', form, config) + .then(() => true) + .catch((error) => { + console.error('Failed to upload item', error) + this.$toast.error(error.response?.data || 'Oops, something went wrong...') + return false + }) + if (!success) allFilesSuccessful = false + } else { + // Chunked upload + const fileId = crypto.randomUUID ? crypto.randomUUID() : Math.random().toString(36).substring(2) + Date.now().toString(36) + const totalChunks = Math.ceil(file.size / CHUNK_SIZE) + + for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) { + const start = chunkIndex * CHUNK_SIZE + const end = Math.min(start + CHUNK_SIZE, file.size) + const chunk = file.slice(start, end) + + var chunkForm = new FormData() + chunkForm.set('fileId', fileId) + chunkForm.set('chunkIndex', chunkIndex) + chunkForm.set('totalChunks', totalChunks) + chunkForm.set('filename', file.name) + chunkForm.set('title', item.title) + if (!this.selectedLibraryIsPodcast) { + chunkForm.set('author', item.author || '') + chunkForm.set('series', item.series || '') + } + chunkForm.set('library', this.selectedLibraryId) + chunkForm.set('folder', this.selectedFolderId) + chunkForm.set('chunk', chunk) + + try { + // Await each chunk sequentially + await this.$axios.$post('/api/upload/chunk', chunkForm) + } catch (error) { + console.error(`Failed to upload chunk ${chunkIndex} for ${file.name}`, error) + allFilesSuccessful = false + break // Stop uploading chunks if one fails } - this.updateItemCardProgress(item.index, progress) } } } - - return this.$axios - .$post('/api/upload', form, config) - .then(() => true) - .catch((error) => { - console.error('Failed to upload item', error) - this.$toast.error(error.response?.data || 'Oops, something went wrong...') - return false - }) + return allFilesSuccessful }, validateItems() { var itemData = [] From 9ac3dae265e4bb89b7259957f38e75e2dc6bc822 Mon Sep 17 00:00:00 2001 From: Josh Roskos Date: Sat, 16 May 2026 21:10:42 -0500 Subject: [PATCH 4/7] Commit 4: UI Progress & Frontend Error Handling --- client/pages/upload/index.vue | 38 ++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/client/pages/upload/index.vue b/client/pages/upload/index.vue index e4655dc75..49bb82dda 100644 --- a/client/pages/upload/index.vue +++ b/client/pages/upload/index.vue @@ -366,13 +366,37 @@ export default { chunkForm.set('folder', this.selectedFolderId) chunkForm.set('chunk', chunk) - try { - // Await each chunk sequentially - await this.$axios.$post('/api/upload/chunk', chunkForm) - } catch (error) { - console.error(`Failed to upload chunk ${chunkIndex} for ${file.name}`, error) - allFilesSuccessful = false - break // Stop uploading chunks if one fails + let retryCount = 0 + const maxRetries = 1 + let chunkSuccess = false + + while (retryCount <= maxRetries && !chunkSuccess) { + try { + const config = { + onUploadProgress: (progressEvent) => { + if (progressEvent.lengthComputable) { + // Calculate overall progress for this specific file + const totalLoaded = chunkIndex * CHUNK_SIZE + progressEvent.loaded + this.updateItemCardProgress(item.index, { loaded: totalLoaded, total: file.size }) + } + } + } + + await this.$axios.$post('/api/upload/chunk', chunkForm, config) + chunkSuccess = true + } catch (error) { + retryCount++ + console.error(`Failed to upload chunk ${chunkIndex} for ${file.name} (Attempt ${retryCount})`, error) + if (retryCount > maxRetries) { + this.$toast.error(`Failed to upload part of ${file.name}. Upload aborted.`) + allFilesSuccessful = false + break + } + } + } + + if (!chunkSuccess) { + break // Break the outer chunk loop if retries exhausted } } } From 5f92babc02ca204e476ef02b295babe80f2ed97d Mon Sep 17 00:00:00 2001 From: Josh Roskos Date: Sat, 16 May 2026 21:11:13 -0500 Subject: [PATCH 5/7] Chore: Fix Node 25 compatibility in jwa and update lockfiles --- package-lock.json | 15 +++++++++++++++ .../libs/jwa/buffer-equal-constant-time/index.js | 11 ++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8950b4903..a2cab8fc0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2113,6 +2113,21 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "devOptional": true }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", diff --git a/server/libs/jwa/buffer-equal-constant-time/index.js b/server/libs/jwa/buffer-equal-constant-time/index.js index 07c142662..56a7252c1 100644 --- a/server/libs/jwa/buffer-equal-constant-time/index.js +++ b/server/libs/jwa/buffer-equal-constant-time/index.js @@ -28,14 +28,19 @@ function bufferEq(a, b) { } bufferEq.install = function () { - Buffer.prototype.equal = SlowBuffer.prototype.equal = function equal(that) { + Buffer.prototype.equal = function equal(that) { return bufferEq(this, that); }; + if (SlowBuffer && SlowBuffer.prototype) { + SlowBuffer.prototype.equal = Buffer.prototype.equal; + } }; var origBufEqual = Buffer.prototype.equal; -var origSlowBufEqual = SlowBuffer.prototype.equal; +var origSlowBufEqual = SlowBuffer && SlowBuffer.prototype ? SlowBuffer.prototype.equal : undefined; bufferEq.restore = function () { Buffer.prototype.equal = origBufEqual; - SlowBuffer.prototype.equal = origSlowBufEqual; + if (SlowBuffer && SlowBuffer.prototype) { + SlowBuffer.prototype.equal = origSlowBufEqual; + } }; From 8673169280efe9d08c414ae78db676e99547dac3 Mon Sep 17 00:00:00 2001 From: Josh Roskos Date: Sat, 16 May 2026 21:30:12 -0500 Subject: [PATCH 6/7] Chore: Add Node 25 compatibility polyfill for SlowBuffer --- index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/index.js b/index.js index 7379322e8..714504bda 100644 --- a/index.js +++ b/index.js @@ -1,3 +1,9 @@ +// Node 25 compatibility: SlowBuffer is removed +const buffer = require('buffer') +if (!buffer.SlowBuffer) { + buffer.SlowBuffer = buffer.Buffer +} + const optionDefinitions = [ { name: 'config', alias: 'c', type: String }, { name: 'metadata', alias: 'm', type: String }, From e1ceac63e25363a9bea070352d9576cd1294413a Mon Sep 17 00:00:00 2001 From: Josh Roskos Date: Sun, 17 May 2026 10:45:45 -0500 Subject: [PATCH 7/7] build and publish workflow --- .github/workflows/build-and-publish.yaml | 73 ++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/build-and-publish.yaml diff --git a/.github/workflows/build-and-publish.yaml b/.github/workflows/build-and-publish.yaml new file mode 100644 index 000000000..d01267a03 --- /dev/null +++ b/.github/workflows/build-and-publish.yaml @@ -0,0 +1,73 @@ +name: CI/CD Pipeline + +on: + push: + branches: + - '5108-chunked-upload-support' + pull_request: + branches: + - '5108-chunked-upload-support' + +env: + REGISTRY: ghcr.io + # This automatically resolves to kc9wwh/audiobookshelf + IMAGE_NAME: ${{ github.repository }} + +jobs: + test-and-build: + name: Test Node Build + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' # Standard for modern ABS + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: npm test --if-present + + docker-publish: + name: Build and Push to GHCR + needs: test-and-build + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # Critical: allows pushing the image to ghcr.io + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=sha + # This creates a tag like ghcr.io/kc9wwh/audiobookshelf:5108-chunked-upload-support + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file