Add chunked, resumable uploads

Split each uploaded file into 5 MB chunks posted sequentially to a new
/api/upload/chunk endpoint, then assemble them on the server via
/api/upload/finalize. The client queries /api/upload/chunk/:uploadId/:fileIndex
to skip chunks already received, so an interrupted upload resumes instead
of restarting. Each chunk is retried with backoff. Path components are
sanitized and the same canUpload/library-access checks as /api/upload are
enforced. The legacy /api/upload endpoint is left intact.

The chunk staging directory defaults to <metadata>/uploads and can be
relocated with UPLOAD_TEMP_DIR so large in-progress uploads can be staged
off the metadata volume.

client/pages/upload/index.vue

@@ -307,37 +307,87 @@ export default {
       }
     },
     async uploadItem(item) {
-      var form = new FormData()
+      const chunkSize = 5 * 1024 * 1024
-      form.set('title', item.title)
+      const uploadId = `${Date.now()}-${Math.random().toString(36).slice(2)}`
-      if (!this.selectedLibraryIsPodcast) {
+      const totalBytes = item.files.reduce((sum, file) => sum + file.size, 0)
-        form.set('author', item.author || '')
+      let sentBytes = 0
-        form.set('series', item.series || '')
-      }
-      form.set('library', this.selectedLibraryId)
-      form.set('folder', this.selectedFolderId)
 
-      var index = 0
+      for (let fileIndex = 0; fileIndex < item.files.length; fileIndex++) {
-      item.files.forEach((file) => {
+        const file = item.files[fileIndex]
-        form.set(`${index++}`, file)
+        const numChunks = Math.max(1, Math.ceil(file.size / chunkSize))
-      })
 
-      const config = {
+        let uploadedChunks = []
-        onUploadProgress: (progressEvent) => {
+        try {
-          if (progressEvent.lengthComputable) {
+          const existing = await this.$axios.$get(`/api/upload/chunk/${uploadId}/${fileIndex}`)
-            const progress = {
+          uploadedChunks = existing.chunks || []
-              loaded: progressEvent.loaded,
+        } catch (error) {
-              total: progressEvent.total
+          uploadedChunks = []
-            }
+        }
-            this.updateItemCardProgress(item.index, progress)
+
+        for (let chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+          const start = chunkIndex * chunkSize
+          const end = Math.min(file.size, start + chunkSize)
+          if (uploadedChunks.includes(chunkIndex)) {
+            sentBytes += end - start
+            this.updateItemCardProgress(item.index, { loaded: sentBytes, total: totalBytes })
+            continue
           }
+
+          const form = new FormData()
+          form.set('uploadId', uploadId)
+          form.set('fileIndex', `${fileIndex}`)
+          form.set('chunkIndex', `${chunkIndex}`)
+          form.set('numChunks', `${numChunks}`)
+          form.set('chunk', file.slice(start, end), file.name)
+
+          let attempt = 0
+          while (true) {
+            try {
+              await this.$axios.$post('/api/upload/chunk', form, {
+                onUploadProgress: (progressEvent) => {
+                  if (progressEvent.lengthComputable) {
+                    this.updateItemCardProgress(item.index, { loaded: sentBytes + progressEvent.loaded, total: totalBytes })
+                  }
+                }
+              })
+              break
+            } catch (error) {
+              attempt++
+              if (attempt >= 5) {
+                console.error('Failed to upload chunk', error)
+                this.$toast.error(error.response?.data || 'Oops, something went wrong...')
+                return false
+              }
+              await new Promise((resolve) => setTimeout(resolve, 1000 * attempt))
+            }
+          }
+
+          sentBytes += end - start
+          this.updateItemCardProgress(item.index, { loaded: sentBytes, total: totalBytes })
         }
       }
 
+      const payload = {
+        uploadId,
+        title: item.title,
+        library: this.selectedLibraryId,
+        folder: this.selectedFolderId,
+        files: item.files.map((file, index) => ({
+          index,
+          name: file.name,
+          numChunks: Math.max(1, Math.ceil(file.size / chunkSize))
+        }))
+      }
+      if (!this.selectedLibraryIsPodcast) {
+        payload.author = item.author || ''
+        payload.series = item.series || ''
+      }
+
       return this.$axios
-        .$post('/api/upload', form, config)
+        .$post('/api/upload/finalize', payload)
         .then(() => true)
         .catch((error) => {
-          console.error('Failed to upload item', error)
+          console.error('Failed to finalize upload', error)
           this.$toast.error(error.response?.data || 'Oops, something went wrong...')
           return false
         })

server/controllers/MiscController.js

@@ -15,6 +15,52 @@ const { sanitizeFilename } = require('../utils/fileUtils')
 const TaskManager = require('../managers/TaskManager')
 const adminStats = require('../utils/queries/adminStats')
 
+function getUploadsRoot() {
+  return process.env.UPLOAD_TEMP_DIR || Path.join(global.MetadataPath, 'uploads')
+}
+
+function getUploadSessionDir(uploadId) {
+  if (!uploadId || typeof uploadId !== 'string' || !/^[a-z0-9-]+$/i.test(uploadId)) {
+    return null
+  }
+  return Path.join(getUploadsRoot(), uploadId)
+}
+
+function getUploadChunkDir(uploadId, fileIndex) {
+  const sessionDir = getUploadSessionDir(uploadId)
+  if (!sessionDir || !/^\d+$/.test(`${fileIndex}`)) {
+    return null
+  }
+  return Path.join(sessionDir, `${Number(fileIndex)}`)
+}
+
+async function assembleChunks(chunkDir, numChunks, destPath) {
+  for (let i = 0; i < numChunks; i++) {
+    if (!(await fs.pathExists(Path.join(chunkDir, `${i}`)))) {
+      return false
+    }
+  }
+  const writeStream = fs.createWriteStream(destPath)
+  try {
+    for (let i = 0; i < numChunks; i++) {
+      const chunkPath = Path.join(chunkDir, `${i}`)
+      await new Promise((resolve, reject) => {
+        const readStream = fs.createReadStream(chunkPath)
+        readStream.on('error', reject)
+        readStream.on('end', resolve)
+        readStream.pipe(writeStream, { end: false })
+      })
+    }
+  } finally {
+    writeStream.end()
+  }
+  await new Promise((resolve, reject) => {
+    writeStream.on('close', resolve)
+    writeStream.on('error', reject)
+  })
+  return true
+}
+
 /**
  * @typedef RequestUserObject
  * @property {import('../models/User')} user
@@ -99,6 +145,124 @@ class MiscController {
     res.sendStatus(200)
   }
 
+  /**
+   * GET: /api/upload/chunk/:uploadId/:fileIndex
+   * List chunk indices already stored for a file, so the client can resume
+   *
+   * @param {RequestWithUser} req
+   * @param {Response} res
+   */
+  async getUploadChunks(req, res) {
+    if (!req.user.canUpload) {
+      return res.sendStatus(403)
+    }
+    const chunkDir = getUploadChunkDir(req.params.uploadId, req.params.fileIndex)
+    if (!chunkDir) {
+      return res.status(400).send('Invalid upload reference')
+    }
+    if (!(await fs.pathExists(chunkDir))) {
+      return res.json({ chunks: [] })
+    }
+    const entries = await fs.readdir(chunkDir)
+    const chunks = entries.map((name) => Number(name)).filter((n) => Number.isInteger(n))
+    res.json({ chunks })
+  }
+
+  /**
+   * POST: /api/upload/chunk
+   * Receive a single chunk of a resumable upload
+   *
+   * @param {RequestWithUser} req
+   * @param {Response} res
+   */
+  async handleUploadChunk(req, res) {
+    if (!req.user.canUpload) {
+      Logger.warn(`User "${req.user.username}" attempted to upload without permission`)
+      return res.sendStatus(403)
+    }
+    if (!req.files || !req.files.chunk) {
+      return res.status(400).send('No chunk in request')
+    }
+    const { uploadId, fileIndex, chunkIndex } = req.body
+    const chunkDir = getUploadChunkDir(uploadId, fileIndex)
+    if (!chunkDir || !/^\d+$/.test(`${chunkIndex}`)) {
+      return res.status(400).send('Invalid chunk request')
+    }
+    await fs.ensureDir(chunkDir)
+    await req.files.chunk.mv(Path.join(chunkDir, `${Number(chunkIndex)}`))
+    res.sendStatus(200)
+  }
+
+  /**
+   * POST: /api/upload/finalize
+   * Assemble all received chunks into files and move them to the library folder
+   *
+   * @param {RequestWithUser} req
+   * @param {Response} res
+   */
+  async handleUploadFinalize(req, res) {
+    if (!req.user.canUpload) {
+      Logger.warn(`User "${req.user.username}" attempted to upload without permission`)
+      return res.sendStatus(403)
+    }
+
+    let { uploadId, title, author, series, folder: folderId, library: libraryId, files } = req.body
+    if (!libraryId || !folderId || typeof libraryId !== 'string' || typeof folderId !== 'string' || !title || typeof title !== 'string' || !Array.isArray(files) || !files.length) {
+      return res.status(400).send('Invalid request body')
+    }
+    const sessionDir = getUploadSessionDir(uploadId)
+    if (!sessionDir) {
+      return res.status(400).send('Invalid upload reference')
+    }
+    if (!series || typeof series !== 'string') {
+      series = null
+    }
+    if (!author || typeof author !== 'string') {
+      author = null
+    }
+
+    const library = await Database.libraryModel.findByIdWithFolders(libraryId)
+    if (!library) {
+      return res.status(404).send('Library not found')
+    }
+
+    if (!req.user.checkCanAccessLibrary(library.id)) {
+      Logger.error(`[MiscController] User "${req.user.username}" attempting to upload to library "${library.id}" without access`)
+      return res.sendStatus(403)
+    }
+
+    const folder = library.libraryFolders.find((fold) => fold.id === folderId)
+    if (!folder) {
+      return res.status(404).send('Folder not found')
+    }
+
+    const outputDirectoryParts = library.isPodcast ? [title] : [author, series, title]
+    const cleanedOutputDirectoryParts = outputDirectoryParts.filter(Boolean).map((part) => sanitizeFilename(part))
+    const outputDirectory = Path.join(...[folder.path, ...cleanedOutputDirectoryParts])
+
+    await fs.ensureDir(outputDirectory)
+
+    Logger.info(`Assembling ${files.length} uploaded files to`, outputDirectory)
+
+    for (const file of files) {
+      const chunkDir = getUploadChunkDir(uploadId, file.index)
+      const numChunks = Number(file.numChunks)
+      if (!chunkDir || !file.name || typeof file.name !== 'string' || !Number.isInteger(numChunks) || numChunks < 1) {
+        await fs.remove(sessionDir)
+        return res.status(400).send('Invalid file entry')
+      }
+      const destPath = Path.join(outputDirectory, sanitizeFilename(file.name))
+      const assembled = await assembleChunks(chunkDir, numChunks, destPath)
+      if (!assembled) {
+        await fs.remove(sessionDir)
+        return res.status(400).send('Missing chunks')
+      }
+    }
+
+    await fs.remove(sessionDir)
+    res.sendStatus(200)
+  }
+
   /**
    * GET: /api/tasks
    * Get tasks for task manager

server/routers/ApiRouter.js

@@ -339,6 +339,9 @@ class ApiRouter {
     // Misc Routes
     //
     this.router.post('/upload', MiscController.handleUpload.bind(this))
+    this.router.post('/upload/chunk', MiscController.handleUploadChunk.bind(this))
+    this.router.get('/upload/chunk/:uploadId/:fileIndex', MiscController.getUploadChunks.bind(this))
+    this.router.post('/upload/finalize', MiscController.handleUploadFinalize.bind(this))
     this.router.get('/tasks', MiscController.getTasks.bind(this))
     this.router.patch('/settings', MiscController.updateServerSettings.bind(this))
     this.router.patch('/sorting-prefixes', MiscController.updateSortingPrefixes.bind(this))

test/server/controllers/MiscController.test.js

@@ -0,0 +1,171 @@
+const { expect } = require('chai')
+const sinon = require('sinon')
+const Path = require('path')
+const os = require('os')
+
+const fs = require('../../../server/libs/fsExtra')
+const Database = require('../../../server/Database')
+const Logger = require('../../../server/Logger')
+const MiscController = require('../../../server/controllers/MiscController')
+
+function mockResponse() {
+  const res = {}
+  res.statusCode = 200
+  res.status = (code) => {
+    res.statusCode = code
+    return res
+  }
+  res.send = (body) => {
+    res.body = body
+    return res
+  }
+  res.sendStatus = (code) => {
+    res.statusCode = code
+    return res
+  }
+  res.json = (obj) => {
+    res.jsonBody = obj
+    return res
+  }
+  return res
+}
+
+function chunkRequest(uploadId, fileIndex, chunkIndex, numChunks, buffer) {
+  return {
+    user: { canUpload: true, username: 'admin', checkCanAccessLibrary: () => true },
+    body: { uploadId, fileIndex: `${fileIndex}`, chunkIndex: `${chunkIndex}`, numChunks: `${numChunks}` },
+    files: {
+      chunk: {
+        mv: async (dest) => fs.writeFile(dest, buffer)
+      }
+    }
+  }
+}
+
+describe('MiscController - chunked resumable upload', () => {
+  let metadataRoot
+  let libraryRoot
+
+  beforeEach(async () => {
+    metadataRoot = await fs.mkdtemp(Path.join(os.tmpdir(), 'abs-meta-'))
+    libraryRoot = await fs.mkdtemp(Path.join(os.tmpdir(), 'abs-lib-'))
+    global.MetadataPath = metadataRoot
+
+    sinon.stub(Database, 'libraryModel').get(() => ({
+      findByIdWithFolders: async (id) => (id === 'lib1' ? { id: 'lib1', isPodcast: false, libraryFolders: [{ id: 'fold1', path: libraryRoot }] } : null)
+    }))
+
+    sinon.stub(Logger, 'info')
+    sinon.stub(Logger, 'warn')
+    sinon.stub(Logger, 'error')
+  })
+
+  afterEach(async () => {
+    sinon.restore()
+    await fs.remove(metadataRoot)
+    await fs.remove(libraryRoot)
+    delete global.MetadataPath
+  })
+
+  it('stores chunks, lists them for resume, then assembles them in order', async () => {
+    const uploadId = 'session-abc'
+
+    let res = mockResponse()
+    await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 0, 2, Buffer.from('hello ')), res)
+    expect(res.statusCode).to.equal(200)
+
+    res = mockResponse()
+    await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 1, 2, Buffer.from('world')), res)
+    expect(res.statusCode).to.equal(200)
+
+    res = mockResponse()
+    await MiscController.getUploadChunks({ user: { canUpload: true }, params: { uploadId, fileIndex: '0' } }, res)
+    expect(res.jsonBody.chunks.sort()).to.deep.equal([0, 1])
+
+    res = mockResponse()
+    await MiscController.handleUploadFinalize(
+      {
+        user: { canUpload: true, username: 'admin', checkCanAccessLibrary: () => true },
+        body: { uploadId, title: 'My Book', author: 'Author', library: 'lib1', folder: 'fold1', files: [{ index: 0, name: 'audio.mp3', numChunks: 2 }] }
+      },
+      res
+    )
+    expect(res.statusCode).to.equal(200)
+
+    const assembled = await fs.readFile(Path.join(libraryRoot, 'Author', 'My Book', 'audio.mp3'))
+    expect(assembled.toString()).to.equal('hello world')
+
+    expect(await fs.pathExists(Path.join(metadataRoot, 'uploads', uploadId))).to.equal(false)
+  })
+
+  it('only re-sends the missing chunk on resume', async () => {
+    const uploadId = 'session-resume'
+
+    let res = mockResponse()
+    await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 0, 2, Buffer.from('AAAA')), res)
+
+    res = mockResponse()
+    await MiscController.getUploadChunks({ user: { canUpload: true }, params: { uploadId, fileIndex: '0' } }, res)
+    expect(res.jsonBody.chunks).to.deep.equal([0])
+
+    res = mockResponse()
+    await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 1, 2, Buffer.from('BBBB')), res)
+
+    res = mockResponse()
+    await MiscController.handleUploadFinalize(
+      {
+        user: { canUpload: true, username: 'admin', checkCanAccessLibrary: () => true },
+        body: { uploadId, title: 'Resumed', library: 'lib1', folder: 'fold1', files: [{ index: 0, name: 'audio.mp3', numChunks: 2 }] }
+      },
+      res
+    )
+    expect(res.statusCode).to.equal(200)
+
+    const assembled = await fs.readFile(Path.join(libraryRoot, 'Resumed', 'audio.mp3'))
+    expect(assembled.toString()).to.equal('AAAABBBB')
+  })
+
+  it('fails finalize when a chunk is missing', async () => {
+    const uploadId = 'session-missing'
+
+    let res = mockResponse()
+    await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 0, 2, Buffer.from('only-first')), res)
+
+    res = mockResponse()
+    await MiscController.handleUploadFinalize(
+      {
+        user: { canUpload: true, username: 'admin', checkCanAccessLibrary: () => true },
+        body: { uploadId, title: 'Broken', library: 'lib1', folder: 'fold1', files: [{ index: 0, name: 'audio.mp3', numChunks: 2 }] }
+      },
+      res
+    )
+    expect(res.statusCode).to.equal(400)
+  })
+
+  it('rejects an upload id that escapes the uploads directory', async () => {
+    const res = mockResponse()
+    await MiscController.handleUploadChunk(chunkRequest('../escape', 0, 0, 1, Buffer.from('x')), res)
+    expect(res.statusCode).to.equal(400)
+  })
+
+  it('rejects a chunk upload without permission', async () => {
+    const res = mockResponse()
+    await MiscController.handleUploadChunk({ user: { canUpload: false, username: 'reader' }, body: {}, files: {} }, res)
+    expect(res.statusCode).to.equal(403)
+  })
+
+  it('stages chunks under UPLOAD_TEMP_DIR when set', async () => {
+    const altRoot = await fs.mkdtemp(Path.join(os.tmpdir(), 'abs-up-'))
+    process.env.UPLOAD_TEMP_DIR = altRoot
+    try {
+      const uploadId = 'session-env'
+      const res = mockResponse()
+      await MiscController.handleUploadChunk(chunkRequest(uploadId, 0, 0, 1, Buffer.from('Z')), res)
+      expect(res.statusCode).to.equal(200)
+      expect(await fs.pathExists(Path.join(altRoot, uploadId, '0', '0'))).to.equal(true)
+    } finally {
+      delete process.env.UPLOAD_TEMP_DIR
+      await fs.remove(altRoot)
+    }
+  })
+})