From 9a76293f20fc6be707b5f76f0983d5e035b1612e Mon Sep 17 00:00:00 2001 From: Carlos Galindo <9221197+kauron@users.noreply.github.com> Date: Mon, 18 May 2026 00:15:05 +0200 Subject: [PATCH] Consider alternative encodings for text files --- package.json | 1 + server/utils/fileUtils.js | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index a0b340244..c65ff537b 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "express-session": "^1.17.3", "graceful-fs": "^4.2.10", "htmlparser2": "^8.0.1", + "jschardet": "^3.1.4", "lru-cache": "^10.0.3", "node-unrar-js": "^2.0.2", "nodemailer": "^6.9.13", diff --git a/server/utils/fileUtils.js b/server/utils/fileUtils.js index 9a349bd54..d4d3ddf7f 100644 --- a/server/utils/fileUtils.js +++ b/server/utils/fileUtils.js @@ -1,6 +1,8 @@ const axios = require('axios') const Path = require('path') const ssrfFilter = require('ssrf-req-filter') +const iconvlite = require('iconv-lite') +const jschardet = require('jschardet') const exec = require('child_process').exec const fs = require('../libs/fsExtra') const rra = require('../libs/recursiveReaddirAsync') @@ -122,7 +124,15 @@ module.exports.getIno = getIno */ async function readTextFile(path) { try { - var data = await fs.readFile(path) + let data = await fs.readFile(path) + const det_enc = jschardet.detect(data) + if (det_enc) + try { + return iconvlite.decode(data, det_enc.encoding) + } catch (error) { + Logger.info(`[FileUtils] ReadTextFile detected encoding ${det_enc.encoding} for file ${path} failed to decode, defaulting to UTF-8. Internal error in iconv-lite: ${error}`) + } + // defaults to utf-8 return String(data) } catch (error) { Logger.error(`[FileUtils] ReadTextFile error ${error}`)