Fix charset decoding for text bodies

lib/read.js

@@ -15,6 +15,7 @@ const createError = require('http-errors')
 const getBody = require('raw-body')
 const iconv = require('iconv-lite')
 const onFinished = require('on-finished')
+const { TextDecoder } = require('node:util')
 const zlib = require('node:zlib')
 const hasBody = require('type-is').hasBody
 const contentType = require('content-type')
@@ -93,7 +94,7 @@ function read (req, res, next, parse, debug, options) {
   }
 
   // assert charset is supported
-  if (verify && encoding !== null && !iconv.encodingExists(encoding)) {
+  if (encoding !== null && !isEncodingSupported(encoding)) {
     return next(createError(415, 'unsupported charset "' + encoding.toUpperCase() + '"', {
       charset: encoding.toLowerCase(),
       type: 'charset.unsupported'
@@ -103,7 +104,7 @@ function read (req, res, next, parse, debug, options) {
   // set raw-body options
   const rawBodyOptions = {
     length,
-    encoding: verify ? null : encoding,
+    encoding: null,
     limit: options.limit
   }
 
@@ -155,9 +156,7 @@ function read (req, res, next, parse, debug, options) {
     let str = body
     try {
       debug('parse body')
-      str = typeof body !== 'string' && encoding !== null
-        ? iconv.decode(body, encoding)
-        : body
+      str = decodeBody(body, encoding)
       req.body = parse(str, encoding)
     } catch (err) {
       next(createError(400, err, {
@@ -171,6 +170,56 @@ function read (req, res, next, parse, debug, options) {
   })
 }
 
+/**
+ * Decode a request body.
+ *
+ * @param {Buffer|string} body
+ * @param {string|null} encoding
+ * @returns {string|Buffer}
+ * @private
+ */
+function decodeBody (body, encoding) {
+  if (typeof body === 'string' || encoding === null) {
+    return body
+  }
+
+  const decoder = getTextDecoder(encoding)
+
+  return decoder !== null
+    ? decoder.decode(body)
+    : iconv.decode(body, encoding)
+}
+
+/**
+ * Create a WHATWG text decoder for the given encoding.
+ *
+ * @param {string} encoding
+ * @returns {TextDecoder|null}
+ * @private
+ */
+function getTextDecoder (encoding) {
+  if (encoding === 'utf-16') {
+    return null
+  }
+
+  try {
+    return new TextDecoder(encoding)
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Determine if an encoding is supported.
+ *
+ * @param {string} encoding
+ * @returns {boolean}
+ * @private
+ */
+function isEncodingSupported (encoding) {
+  return getTextDecoder(encoding) !== null || iconv.encodingExists(encoding)
+}
+
 /**
  * Get the content stream of the request.
  *

test/text.js

@@ -461,6 +461,28 @@ describe('bodyParser.text()', function () {
       test.expect(200, '"name is нет"', done)
     })
 
+    it('should decode windows-1252 C1 controls without replacement characters', function (done) {
+      const test = request(this.server).post('/')
+      test.set('Content-Type', 'text/plain; charset=windows-1252')
+      test.write(Buffer.from('808182838d9e9f', 'hex'))
+      test.expect(200)
+      test.expect(function (res) {
+        assert.strictEqual(JSON.parse(res.text), '€\x81‚ƒ\x8džŸ')
+      })
+      test.end(done)
+    })
+
+    it('should replace invalid utf-16 surrogate pairs', function (done) {
+      const test = request(this.server).post('/')
+      test.set('Content-Type', 'text/plain; charset=utf-16le')
+      test.write(Buffer.from('00d800d8', 'hex'))
+      test.expect(200)
+      test.expect(function (res) {
+        assert.strictEqual(JSON.parse(res.text), '\ufffd\ufffd')
+      })
+      test.end(done)
+    })
+
     it('should parse when content-length != char length', function (done) {
       const test = request(this.server).post('/')
       test.set('Content-Type', 'text/plain; charset=utf-8')