diff --git a/client/pages/upload/index.vue b/client/pages/upload/index.vue index adc21ff90..5b89d35a5 100644 --- a/client/pages/upload/index.vue +++ b/client/pages/upload/index.vue @@ -307,37 +307,87 @@ export default { } }, async uploadItem(item) { - var form = new FormData() - form.set('title', item.title) - if (!this.selectedLibraryIsPodcast) { - form.set('author', item.author || '') - form.set('series', item.series || '') - } - form.set('library', this.selectedLibraryId) - form.set('folder', this.selectedFolderId) + const chunkSize = 5 * 1024 * 1024 + const uploadId = `${Date.now()}-${Math.random().toString(36).slice(2)}` + const totalBytes = item.files.reduce((sum, file) => sum + file.size, 0) + let sentBytes = 0 - var index = 0 - item.files.forEach((file) => { - form.set(`${index++}`, file) - }) + for (let fileIndex = 0; fileIndex < item.files.length; fileIndex++) { + const file = item.files[fileIndex] + const numChunks = Math.max(1, Math.ceil(file.size / chunkSize)) - const config = { - onUploadProgress: (progressEvent) => { - if (progressEvent.lengthComputable) { - const progress = { - loaded: progressEvent.loaded, - total: progressEvent.total - } - this.updateItemCardProgress(item.index, progress) + let uploadedChunks = [] + try { + const existing = await this.$axios.$get(`/api/upload/chunk/${uploadId}/${fileIndex}`) + uploadedChunks = existing.chunks || [] + } catch (error) { + uploadedChunks = [] + } + + for (let chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) { + const start = chunkIndex * chunkSize + const end = Math.min(file.size, start + chunkSize) + if (uploadedChunks.includes(chunkIndex)) { + sentBytes += end - start + this.updateItemCardProgress(item.index, { loaded: sentBytes, total: totalBytes }) + continue } + + const form = new FormData() + form.set('uploadId', uploadId) + form.set('fileIndex', `${fileIndex}`) + form.set('chunkIndex', `${chunkIndex}`) + form.set('numChunks', `${numChunks}`) + form.set('chunk', file.slice(start, end), file.name) + + let attempt = 0 + while (true) { + try { + await this.$axios.$post('/api/upload/chunk', form, { + onUploadProgress: (progressEvent) => { + if (progressEvent.lengthComputable) { + this.updateItemCardProgress(item.index, { loaded: sentBytes + progressEvent.loaded, total: totalBytes }) + } + } + }) + break + } catch (error) { + attempt++ + if (attempt >= 5) { + console.error('Failed to upload chunk', error) + this.$toast.error(error.response?.data || 'Oops, something went wrong...') + return false + } + await new Promise((resolve) => setTimeout(resolve, 1000 * attempt)) + } + } + + sentBytes += end - start + this.updateItemCardProgress(item.index, { loaded: sentBytes, total: totalBytes }) } } + const payload = { + uploadId, + title: item.title, + library: this.selectedLibraryId, + folder: this.selectedFolderId, + files: item.files.map((file, index) => ({ + index, + name: file.name, + numChunks: Math.max(1, Math.ceil(file.size / chunkSize)) + })) + } + if (!this.selectedLibraryIsPodcast) { + payload.author = item.author || '' + payload.series = item.series || '' + } + return this.$axios - .$post('/api/upload', form, config) + .$post('/api/upload/finalize', payload) .then(() => true) .catch((error) => { - console.error('Failed to upload item', error) + console.error('Failed to finalize upload', error) this.$toast.error(error.response?.data || 'Oops, something went wrong...') return false }) diff --git a/server/controllers/MiscController.js b/server/controllers/MiscController.js index 591f8ccf2..9498141bf 100644 --- a/server/controllers/MiscController.js +++ b/server/controllers/MiscController.js @@ -15,6 +15,52 @@ const { sanitizeFilename } = require('../utils/fileUtils') const TaskManager = require('../managers/TaskManager') const adminStats = require('../utils/queries/adminStats') +function getUploadsRoot() { + return process.env.UPLOAD_TEMP_DIR || Path.join(global.MetadataPath, 'uploads') +} + +function getUploadSessionDir(uploadId) { + if (!uploadId || typeof uploadId !== 'string' || !/^[a-z0-9-]+$/i.test(uploadId)) { + return null + } + return Path.join(getUploadsRoot(), uploadId) +} + +function getUploadChunkDir(uploadId, fileIndex) { + const sessionDir = getUploadSessionDir(uploadId) + if (!sessionDir || !/^\d+$/.test(`${fileIndex}`)) { + return null + } + return Path.join(sessionDir, `${Number(fileIndex)}`) +} + +async function assembleChunks(chunkDir, numChunks, destPath) { + for (let i = 0; i < numChunks; i++) { + if (!(await fs.pathExists(Path.join(chunkDir, `${i}`)))) { + return false + } + } + const writeStream = fs.createWriteStream(destPath) + try { + for (let i = 0; i < numChunks; i++) { + const chunkPath = Path.join(chunkDir, `${i}`) + await new Promise((resolve, reject) => { + const readStream = fs.createReadStream(chunkPath) + readStream.on('error', reject) + readStream.on('end', resolve) + readStream.pipe(writeStream, { end: false }) + }) + } + } finally { + writeStream.end() + } + await new Promise((resolve, reject) => { + writeStream.on('close', resolve) + writeStream.on('error', reject) + }) + return true +} + /** * @typedef RequestUserObject * @property {import('../models/User')} user @@ -99,6 +145,124 @@ class MiscController { res.sendStatus(200) } + /** + * GET: /api/upload/chunk/:uploadId/:fileIndex + * List chunk indices already stored for a file, so the client can resume + * + * @param {RequestWithUser} req + * @param {Response} res + */ + async getUploadChunks(req, res) { + if (!req.user.canUpload) { + return res.sendStatus(403) + } + const chunkDir = getUploadChunkDir(req.params.uploadId, req.params.fileIndex) + if (!chunkDir) { + return res.status(400).send('Invalid upload reference') + } + if (!(await fs.pathExists(chunkDir))) { + return res.json({ chunks: [] }) + } + const entries = await fs.readdir(chunkDir) + const chunks = entries.map((name) => Number(name)).filter((n) => Number.isInteger(n)) + res.json({ chunks }) + } + + /** + * POST: /api/upload/chunk + * Receive a single chunk of a resumable upload + * + * @param {RequestWithUser} req + * @param {Response} res + */ + async handleUploadChunk(req, res) { + if (!req.user.canUpload) { + Logger.warn(`User "${req.user.username}" attempted to upload without permission`) + return res.sendStatus(403) + } + if (!req.files || !req.files.chunk) { + return res.status(400).send('No chunk in request') + } + const { uploadId, fileIndex, chunkIndex } = req.body + const chunkDir = getUploadChunkDir(uploadId, fileIndex) + if (!chunkDir || !/^\d+$/.test(`${chunkIndex}`)) { + return res.status(400).send('Invalid chunk request') + } + await fs.ensureDir(chunkDir) + await req.files.chunk.mv(Path.join(chunkDir, `${Number(chunkIndex)}`)) + res.sendStatus(200) + } + + /** + * POST: /api/upload/finalize + * Assemble all received chunks into files and move them to the library folder + * + * @param {RequestWithUser} req + * @param {Response} res + */ + async handleUploadFinalize(req, res) { + if (!req.user.canUpload) { + Logger.warn(`User "${req.user.username}" attempted to upload without permission`) + return res.sendStatus(403) + } + + let { uploadId, title, author, series, folder: folderId, library: libraryId, files } = req.body + if (!libraryId || !folderId || typeof libraryId !== 'string' || typeof folderId !== 'string' || !title || typeof title !== 'string' || !Array.isArray(files) || !files.length) { + return res.status(400).send('Invalid request body') + } + const sessionDir = getUploadSessionDir(uploadId) + if (!sessionDir) { + return res.status(400).send('Invalid upload reference') + } + if (!series || typeof series !== 'string') { + series = null + } + if (!author || typeof author !== 'string') { + author = null + } + + const library = await Database.libraryModel.findByIdWithFolders(libraryId) + if (!library) { + return res.status(404).send('Library not found') + } + + if (!req.user.checkCanAccessLibrary(library.id)) { + Logger.error(`[MiscController] User "${req.user.username}" attempting to upload to library "${library.id}" without access`) + return res.sendStatus(403) + } + + const folder = library.libraryFolders.find((fold) => fold.id === folderId) + if (!folder) { + return res.status(404).send('Folder not found') + } + + const outputDirectoryParts = library.isPodcast ? [title] : [author, series, title] + const cleanedOutputDirectoryParts = outputDirectoryParts.filter(Boolean).map((part) => sanitizeFilename(part)) + const outputDirectory = Path.join(...[folder.path, ...cleanedOutputDirectoryParts]) + + await fs.ensureDir(outputDirectory) + + Logger.info(`Assembling ${files.length} uploaded files to`, outputDirectory) + + for (const file of files) { + const chunkDir = getUploadChunkDir(uploadId, file.index) + const numChunks = Number(file.numChunks) + if (!chunkDir || !file.name || typeof file.name !== 'string' || !Number.isInteger(numChunks) || numChunks < 1) { + await fs.remove(sessionDir) + return res.status(400).send('Invalid file entry') + } + const destPath = Path.join(outputDirectory, sanitizeFilename(file.name)) + const assembled = await assembleChunks(chunkDir, numChunks, destPath) + if (!assembled) { + await fs.remove(sessionDir) + return res.status(400).send('Missing chunks') + } + } + + await fs.remove(sessionDir) + res.sendStatus(200) + } + /** * GET: /api/tasks * Get tasks for task manager diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index e89b364f3..7884f4cea 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -339,6 +339,9 @@ class ApiRouter { // Misc Routes // this.router.post('/upload', MiscController.handleUpload.bind(this)) + this.router.post('/upload/chunk', MiscController.handleUploadChunk.bind(this)) + this.router.get('/upload/chunk/:uploadId/:fileIndex', MiscController.getUploadChunks.bind(this)) + this.router.post('/upload/finalize', MiscController.handleUploadFinalize.bind(this)) this.router.get('/tasks', MiscController.getTasks.bind(this)) this.router.patch('/settings', MiscController.updateServerSettings.bind(this)) this.router.patch('/sorting-prefixes', MiscController.updateSortingPrefixes.bind(this)) diff --git a/test/server/controllers/MiscController.test.js b/test/server/controllers/MiscController.test.js new file mode 100644 index 000000000..d76a192be --- /dev/null +++ b/test/server/controllers/MiscController.test.js @@ -0,0 +1,171 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const Path = require('path') +const os = require('os') + +const fs = require('../../../server/libs/fsExtra') +const Database = require('../../../server/Database') +const Logger = require('../../../server/Logger') +const MiscController = require('../../../server/controllers/MiscController') + +function mockResponse() { + const res = {} + res.statusCode = 200 + res.status = (code) => { + res.statusCode = code + return res + } + res.send = (body) => { + res.body = body + return res + } + res.sendStatus = (code) => { + res.statusCode = code + return res + } + res.json = (obj) => { + res.jsonBody = obj + return res + } + return res +} + +function chunkRequest(uploadId, fileIndex, chunkIndex, numChunks, buffer) { + return { + user: { canUpload: true, username: 'admin', checkCanAccessLibrary: () => true }, + body: { uploadId, fileIndex: `${fileIndex}`, chunkIndex: `${chunkIndex}`, numChunks: `${numChunks}` }, + files: { + chunk: { + mv: async (dest) => fs.writeFile(dest, buffer) + } + } + } +} + +describe('MiscController - chunked resumable upload', () => { + let metadataRoot + let libraryRoot + + beforeEach(async () => { + metadataRoot = await fs.mkdtemp(Path.join(os.tmpdir(), 'abs-meta-')) + libraryRoot = await fs.mkdtemp(Path.join(os.tmpdir(), 'abs-lib-')) + global.MetadataPath = metadataRoot + + sinon.stub(Database, 'libraryModel').get(() => ({ + findByIdWithFolders: async (id) => (id === 'lib1' ? { id: 'lib1', isPodcast: false, libraryFolders: [{ id: 'fold1', path: libraryRoot }] } : null) + })) + + sinon.stub(Logger, 'info') + sinon.stub(Logger, 'warn') + sinon.stub(Logger, 'error') + }) + + afterEach(async () => { + sinon.restore() + await fs.remove(metadataRoot) + await fs.remove(libraryRoot) + delete global.MetadataPath + }) + + it('stores chunks, lists them for resume, then assembles them in order', async () => { + const uploadId = 'session-abc' + + let res = mockResponse() + await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 0, 2, Buffer.from('hello ')), res) + expect(res.statusCode).to.equal(200) + + res = mockResponse() + await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 1, 2, Buffer.from('world')), res) + expect(res.statusCode).to.equal(200) + + res = mockResponse() + await MiscController.getUploadChunks({ user: { canUpload: true }, params: { uploadId, fileIndex: '0' } }, res) + expect(res.jsonBody.chunks.sort()).to.deep.equal([0, 1]) + + res = mockResponse() + await MiscController.handleUploadFinalize( + { + user: { canUpload: true, username: 'admin', checkCanAccessLibrary: () => true }, + body: { uploadId, title: 'My Book', author: 'Author', library: 'lib1', folder: 'fold1', files: [{ index: 0, name: 'audio.mp3', numChunks: 2 }] } + }, + res + ) + expect(res.statusCode).to.equal(200) + + const assembled = await fs.readFile(Path.join(libraryRoot, 'Author', 'My Book', 'audio.mp3')) + expect(assembled.toString()).to.equal('hello world') + + expect(await fs.pathExists(Path.join(metadataRoot, 'uploads', uploadId))).to.equal(false) + }) + + it('only re-sends the missing chunk on resume', async () => { + const uploadId = 'session-resume' + + let res = mockResponse() + await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 0, 2, Buffer.from('AAAA')), res) + + res = mockResponse() + await MiscController.getUploadChunks({ user: { canUpload: true }, params: { uploadId, fileIndex: '0' } }, res) + expect(res.jsonBody.chunks).to.deep.equal([0]) + + res = mockResponse() + await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 1, 2, Buffer.from('BBBB')), res) + + res = mockResponse() + await MiscController.handleUploadFinalize( + { + user: { canUpload: true, username: 'admin', checkCanAccessLibrary: () => true }, + body: { uploadId, title: 'Resumed', library: 'lib1', folder: 'fold1', files: [{ index: 0, name: 'audio.mp3', numChunks: 2 }] } + }, + res + ) + expect(res.statusCode).to.equal(200) + + const assembled = await fs.readFile(Path.join(libraryRoot, 'Resumed', 'audio.mp3')) + expect(assembled.toString()).to.equal('AAAABBBB') + }) + + it('fails finalize when a chunk is missing', async () => { + const uploadId = 'session-missing' + + let res = mockResponse() + await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 0, 2, Buffer.from('only-first')), res) + + res = mockResponse() + await MiscController.handleUploadFinalize( + { + user: { canUpload: true, username: 'admin', checkCanAccessLibrary: () => true }, + body: { uploadId, title: 'Broken', library: 'lib1', folder: 'fold1', files: [{ index: 0, name: 'audio.mp3', numChunks: 2 }] } + }, + res + ) + expect(res.statusCode).to.equal(400) + }) + + it('rejects an upload id that escapes the uploads directory', async () => { + const res = mockResponse() + await MiscController.handleUploadChunk(chunkRequest('../escape', 0, 0, 1, Buffer.from('x')), res) + expect(res.statusCode).to.equal(400) + }) + + it('rejects a chunk upload without permission', async () => { + const res = mockResponse() + await MiscController.handleUploadChunk({ user: { canUpload: false, username: 'reader' }, body: {}, files: {} }, res) + expect(res.statusCode).to.equal(403) + }) + + it('stages chunks under UPLOAD_TEMP_DIR when set', async () => { + const altRoot = await fs.mkdtemp(Path.join(os.tmpdir(), 'abs-up-')) + process.env.UPLOAD_TEMP_DIR = altRoot + try { + const uploadId = 'session-env' + const res = mockResponse() + await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 0, 1, Buffer.from('Z')), res) + expect(res.statusCode).to.equal(200) + expect(await fs.pathExists(Path.join(altRoot, uploadId, '0', '0'))).to.equal(true) + } finally { + delete process.env.UPLOAD_TEMP_DIR + await fs.remove(altRoot) + } + }) +})