diff --git a/.github/workflows/build-and-publish.yaml b/.github/workflows/build-and-publish.yaml new file mode 100644 index 000000000..d01267a03 --- /dev/null +++ b/.github/workflows/build-and-publish.yaml @@ -0,0 +1,73 @@ +name: CI/CD Pipeline + +on: + push: + branches: + - '5108-chunked-upload-support' + pull_request: + branches: + - '5108-chunked-upload-support' + +env: + REGISTRY: ghcr.io + # This automatically resolves to kc9wwh/audiobookshelf + IMAGE_NAME: ${{ github.repository }} + +jobs: + test-and-build: + name: Test Node Build + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' # Standard for modern ABS + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: npm test --if-present + + docker-publish: + name: Build and Push to GHCR + needs: test-and-build + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # Critical: allows pushing the image to ghcr.io + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=sha + # This creates a tag like ghcr.io/kc9wwh/audiobookshelf:5108-chunked-upload-support + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/client/pages/upload/index.vue b/client/pages/upload/index.vue index adc21ff90..49bb82dda 100644 --- a/client/pages/upload/index.vue +++ b/client/pages/upload/index.vue @@ -307,40 +307,101 @@ export default { } }, async uploadItem(item) { - var form = new FormData() - form.set('title', item.title) - if (!this.selectedLibraryIsPodcast) { - form.set('author', item.author || '') - form.set('series', item.series || '') - } - form.set('library', this.selectedLibraryId) - form.set('folder', this.selectedFolderId) + const CHUNK_SIZE = 50 * 1024 * 1024 // 50MB + let allFilesSuccessful = true - var index = 0 - item.files.forEach((file) => { - form.set(`${index++}`, file) - }) + for (let fileIndex = 0; fileIndex < item.files.length; fileIndex++) { + const file = item.files[fileIndex] - const config = { - onUploadProgress: (progressEvent) => { - if (progressEvent.lengthComputable) { - const progress = { - loaded: progressEvent.loaded, - total: progressEvent.total + if (file.size <= CHUNK_SIZE) { + // Legacy upload for small files + var form = new FormData() + form.set('title', item.title) + if (!this.selectedLibraryIsPodcast) { + form.set('author', item.author || '') + form.set('series', item.series || '') + } + form.set('library', this.selectedLibraryId) + form.set('folder', this.selectedFolderId) + form.set(`${fileIndex}`, file) + + const config = { + onUploadProgress: (progressEvent) => { + if (progressEvent.lengthComputable) { + this.updateItemCardProgress(item.index, { loaded: progressEvent.loaded, total: progressEvent.total }) + } + } + } + + const success = await this.$axios + .$post('/api/upload', form, config) + .then(() => true) + .catch((error) => { + console.error('Failed to upload item', error) + this.$toast.error(error.response?.data || 'Oops, something went wrong...') + return false + }) + if (!success) allFilesSuccessful = false + } else { + // Chunked upload + const fileId = crypto.randomUUID ? crypto.randomUUID() : Math.random().toString(36).substring(2) + Date.now().toString(36) + const totalChunks = Math.ceil(file.size / CHUNK_SIZE) + + for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) { + const start = chunkIndex * CHUNK_SIZE + const end = Math.min(start + CHUNK_SIZE, file.size) + const chunk = file.slice(start, end) + + var chunkForm = new FormData() + chunkForm.set('fileId', fileId) + chunkForm.set('chunkIndex', chunkIndex) + chunkForm.set('totalChunks', totalChunks) + chunkForm.set('filename', file.name) + chunkForm.set('title', item.title) + if (!this.selectedLibraryIsPodcast) { + chunkForm.set('author', item.author || '') + chunkForm.set('series', item.series || '') + } + chunkForm.set('library', this.selectedLibraryId) + chunkForm.set('folder', this.selectedFolderId) + chunkForm.set('chunk', chunk) + + let retryCount = 0 + const maxRetries = 1 + let chunkSuccess = false + + while (retryCount <= maxRetries && !chunkSuccess) { + try { + const config = { + onUploadProgress: (progressEvent) => { + if (progressEvent.lengthComputable) { + // Calculate overall progress for this specific file + const totalLoaded = chunkIndex * CHUNK_SIZE + progressEvent.loaded + this.updateItemCardProgress(item.index, { loaded: totalLoaded, total: file.size }) + } + } + } + + await this.$axios.$post('/api/upload/chunk', chunkForm, config) + chunkSuccess = true + } catch (error) { + retryCount++ + console.error(`Failed to upload chunk ${chunkIndex} for ${file.name} (Attempt ${retryCount})`, error) + if (retryCount > maxRetries) { + this.$toast.error(`Failed to upload part of ${file.name}. Upload aborted.`) + allFilesSuccessful = false + break + } + } + } + + if (!chunkSuccess) { + break // Break the outer chunk loop if retries exhausted } - this.updateItemCardProgress(item.index, progress) } } } - - return this.$axios - .$post('/api/upload', form, config) - .then(() => true) - .catch((error) => { - console.error('Failed to upload item', error) - this.$toast.error(error.response?.data || 'Oops, something went wrong...') - return false - }) + return allFilesSuccessful }, validateItems() { var itemData = [] diff --git a/index.js b/index.js index ce0195e95..8aed5b184 100644 --- a/index.js +++ b/index.js @@ -1,3 +1,9 @@ +// Node 25 compatibility: SlowBuffer is removed +const buffer = require('buffer') +if (!buffer.SlowBuffer) { + buffer.SlowBuffer = buffer.Buffer +} + const optionDefinitions = [ { name: 'config', alias: 'c', type: String }, { name: 'metadata', alias: 'm', type: String }, diff --git a/package-lock.json b/package-lock.json index 9cee15b25..64607fc7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2113,6 +2113,21 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "devOptional": true }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", diff --git a/server/controllers/MiscController.js b/server/controllers/MiscController.js index 591f8ccf2..cd2d95702 100644 --- a/server/controllers/MiscController.js +++ b/server/controllers/MiscController.js @@ -99,6 +99,91 @@ class MiscController { res.sendStatus(200) } + /** + * POST: /api/upload/chunk + * Handle chunked upload + * + * @param {RequestWithUser} req + * @param {Response} res + */ + async handleChunkUpload(req, res) { + if (!req.user.canUpload) { + Logger.warn(`User "${req.user.username}" attempted to upload without permission`) + return res.sendStatus(403) + } + if (!req.files || !Object.values(req.files).length) { + Logger.error('Invalid request, no files') + return res.sendStatus(400) + } + + const { fileId, chunkIndex, totalChunks, filename, library: libraryId, folder: folderId, title, author, series } = req.body + + if (!fileId || chunkIndex === undefined || !totalChunks || !filename || !libraryId || !folderId || !title) { + return res.status(400).send('Invalid request body for chunk upload') + } + + const library = await Database.libraryModel.findByIdWithFolders(libraryId) + if (!library) { + return res.status(404).send('Library not found') + } + if (!req.user.checkCanAccessLibrary(library.id)) { + Logger.error(`[MiscController] User "${req.user.username}" attempting to upload to library "${library.id}" without access`) + return res.sendStatus(403) + } + + const tmpDir = Path.join(global.MetadataPath, 'tmp', 'uploads', fileId) + await fs.ensureDir(tmpDir) + + const file = Object.values(req.files)[0] + const chunkPath = Path.join(tmpDir, `${fileId}_${chunkIndex}`) + + try { + await file.mv(chunkPath) + + // Reassembly logic + if (parseInt(chunkIndex) === parseInt(totalChunks) - 1) { + const folder = library.libraryFolders.find((fold) => fold.id === folderId) + if (!folder) return res.status(404).send('Folder not found') + + const outputDirectoryParts = library.isPodcast ? [title] : [author, series, title] + const cleanedOutputDirectoryParts = outputDirectoryParts.filter(Boolean).map((part) => sanitizeFilename(part)) + const outputDirectory = Path.join(...[folder.path, ...cleanedOutputDirectoryParts]) + await fs.ensureDir(outputDirectory) + + const finalFilePath = Path.join(outputDirectory, sanitizeFilename(filename)) + const writeStream = fs.createWriteStream(finalFilePath) + + for (let i = 0; i < totalChunks; i++) { + const currentChunkPath = Path.join(tmpDir, `${fileId}_${i}`) + const data = await fs.readFile(currentChunkPath) + if (!writeStream.write(data)) { + await new Promise((resolve) => writeStream.once('drain', resolve)) + } + } + + writeStream.end() + + writeStream.on('finish', async () => { + Logger.info(`Successfully merged ${totalChunks} chunks for file ${filename}`) + await fs.remove(tmpDir) // Cleanup + res.sendStatus(200) + }) + + writeStream.on('error', async (error) => { + Logger.error(`Error merging chunks for file ${filename}`, error) + await fs.remove(tmpDir).catch((e) => Logger.error('Failed to clean up temp dir on merge error', e)) // Cleanup on error + res.status(500).send('Error merging file') + }) + } else { + res.sendStatus(200) // Chunk saved, waiting for more + } + } catch (error) { + Logger.error(`Failed to move chunk ${chunkIndex} for file ${fileId}`, error) + await fs.remove(tmpDir).catch((e) => Logger.error('Failed to clean up temp dir on chunk error', e)) // Cleanup + return res.status(500).send('Failed to save chunk') + } + } + /** * GET: /api/tasks * Get tasks for task manager diff --git a/server/libs/jwa/buffer-equal-constant-time/index.js b/server/libs/jwa/buffer-equal-constant-time/index.js index 07c142662..56a7252c1 100644 --- a/server/libs/jwa/buffer-equal-constant-time/index.js +++ b/server/libs/jwa/buffer-equal-constant-time/index.js @@ -28,14 +28,19 @@ function bufferEq(a, b) { } bufferEq.install = function () { - Buffer.prototype.equal = SlowBuffer.prototype.equal = function equal(that) { + Buffer.prototype.equal = function equal(that) { return bufferEq(this, that); }; + if (SlowBuffer && SlowBuffer.prototype) { + SlowBuffer.prototype.equal = Buffer.prototype.equal; + } }; var origBufEqual = Buffer.prototype.equal; -var origSlowBufEqual = SlowBuffer.prototype.equal; +var origSlowBufEqual = SlowBuffer && SlowBuffer.prototype ? SlowBuffer.prototype.equal : undefined; bufferEq.restore = function () { Buffer.prototype.equal = origBufEqual; - SlowBuffer.prototype.equal = origSlowBufEqual; + if (SlowBuffer && SlowBuffer.prototype) { + SlowBuffer.prototype.equal = origSlowBufEqual; + } }; diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index e89b364f3..95de5c957 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -339,6 +339,7 @@ class ApiRouter { // Misc Routes // this.router.post('/upload', MiscController.handleUpload.bind(this)) + this.router.post('/upload/chunk', MiscController.handleChunkUpload.bind(this)) this.router.get('/tasks', MiscController.getTasks.bind(this)) this.router.patch('/settings', MiscController.updateServerSettings.bind(this)) this.router.patch('/sorting-prefixes', MiscController.updateSortingPrefixes.bind(this))