Fix slow downloads: larger buffers, Content-Length headers, parallel streams

This commit is contained in:
Trevor Renshaw 2026-05-16 08:50:36 -06:00
parent c010f0e1eb
commit cf0a432027
3 changed files with 22 additions and 4 deletions

View file

@ -318,8 +318,9 @@ module.exports.downloadFile = (url, filepath, contentTypeFilter = null) => {
const totalSize = parseInt(response.headers['content-length'], 10)
let downloadedSize = 0
// Write to filepath
const writer = fs.createWriteStream(filepath)
// Use a 512 KiB write buffer. Node's default 16 KiB highWaterMark causes
// many small write syscalls for large audio files and limits throughput.
const writer = fs.createWriteStream(filepath, { highWaterMark: 524288 })
response.data.pipe(writer)
let lastProgress = 0

View file

@ -9,7 +9,11 @@ module.exports.zipDirectoryPipe = (path, filename, res) => {
res.attachment(filename)
const archive = archiver('zip', {
zlib: { level: 0 } // Sets the compression level.
zlib: { level: 0 }, // Sets the compression level (0 = store, no CPU overhead).
// Increase the internal pipeline highWaterMark from 16 KiB to 512 KiB.
// This allows archiver to buffer more data before applying backpressure,
// keeping the read-from-disk and write-to-network pipelines busy.
highWaterMark: 524288
})
// listen for all archive data to be written
@ -67,7 +71,9 @@ module.exports.zipDirectoriesPipe = (pathObjects, filename, res) => {
res.attachment(filename)
const archive = archiver('zip', {
zlib: { level: 0 } // Sets the compression level.
zlib: { level: 0 }, // Sets the compression level (0 = store, no CPU overhead).
// Increase the internal pipeline highWaterMark from 16 KiB to 512 KiB.
highWaterMark: 524288
})
// listen for all archive data to be written