fix: prevent double decompression in node:http by disabling fetch auto-decode

The node:http ClientRequest uses fetch() internally, which auto-decompresses
gzip/brotli responses by default. However, it preserves the Content-Encoding
header, causing callers that follow the Node.js convention of manually handling
Content-Encoding to attempt double decompression.

Pass encodeResponseBody: 'manual' to fetch() so raw compressed bytes are
delivered to IncomingMessage, matching Node.js http behavior where callers
handle decompression themselves.

Author: petebacondarwin

src/node/internal/internal_http_client.ts

@@ -365,24 +365,29 @@ export class ClientRequest extends OutgoingMessage implements _ClientRequest {
       url = new URL(this.path, url);
     }
 
-    // Our fetch implementation has the following limitations.
+    // Our fetch implementation has the following limitation:
     //
-    // 1. Content decoding is handled automatically by fetch,
-    //    but expectation is that it's not handled in http.
-    // 2. Nothing is directly waiting for fetch promise here.
-    //    It's up to the user of the HTTP API to arrange for
-    //    the request to be held open until the fetch completes,
-    //    typically by passing some promise to ctx.waitUntil()
-    //    and resolving that promise when the request is complete.
+    // Nothing is directly waiting for fetch promise here.
+    // It's up to the user of the HTTP API to arrange for
+    // the request to be held open until the fetch completes,
+    // typically by passing some promise to ctx.waitUntil()
+    // and resolving that promise when the request is complete.
     //
-    // TODO(soon): Address these concerns and limitations.
+    // TODO(soon): Address this limitation.
+
+    // We use encodeResponseBody: 'manual' to prevent fetch from automatically
+    // decompressing the response body. Node.js http does not auto-decompress;
+    // callers are expected to handle Content-Encoding themselves.
+    // The type assertion is needed because the DOM RequestInit type does not
+    // include the workerd-specific encodeResponseBody property.
     fetch(url, {
       method: this.method,
       headers,
       body: body ?? null,
       signal: this.#abortController.signal,
       redirect: 'manual',
-    })
+      encodeResponseBody: 'manual',
+    } as unknown as RequestInit)
       .then(this.#handleFetchResponse.bind(this))
       .catch(this.#handleFetchError.bind(this));
   }

src/workerd/api/node/tests/BUILD.bazel

@@ -510,6 +510,7 @@ wd_test(
         "DEFAULT_HEADERS_EXIST_PORT",
         "REQUEST_ARGUMENTS_PORT",
         "HELLO_WORLD_SERVER_PORT",
+        "GZIP_SERVER_PORT",
     ],
 )
 

src/workerd/api/node/tests/http-client-nodejs-server.js

@@ -6,6 +6,7 @@
 // It is executed using the appropriate Node.js version defined in build/deps/nodejs.MODULE.bazel.
 const http = require('node:http');
 const assert = require('node:assert/strict');
+const zlib = require('node:zlib');
 
 function listenTo(server, port) {
   server.listen(port, process.env.SIDECAR_HOSTNAME, () => {
@@ -120,3 +121,14 @@ const helloWorldServer = http.createServer((req, res) => {
 });
 
 listenTo(helloWorldServer, process.env.HELLO_WORLD_SERVER_PORT);
+
+const gzipServer = http.createServer((_req, res) => {
+  const body = zlib.gzipSync(Buffer.from('hello from gzip server'));
+  res.writeHead(200, {
+    'Content-Encoding': 'gzip',
+    'Content-Type': 'text/plain',
+  });
+  res.end(body);
+});
+
+listenTo(gzipServer, process.env.GZIP_SERVER_PORT);

src/workerd/api/node/tests/http-client-nodejs-test.js

@@ -6,6 +6,7 @@ import http from 'node:http';
 import { strictEqual, ok, throws, deepStrictEqual } from 'node:assert';
 import { once } from 'node:events';
 import { get } from 'node:http';
+import { gunzipSync } from 'node:zlib';
 
 export const checkPortsSetCorrectly = {
   test(_ctrl, env) {
@@ -16,6 +17,7 @@ export const checkPortsSetCorrectly = {
       'DEFAULT_HEADERS_EXIST_PORT',
       'REQUEST_ARGUMENTS_PORT',
       'HELLO_WORLD_SERVER_PORT',
+      'GZIP_SERVER_PORT',
     ];
     for (const key of keys) {
       strictEqual(typeof env[key], 'string');
@@ -529,6 +531,38 @@ export const testBodyDataDuplicationRegression = {
   },
 };
 
+// Verify that the http module does not automatically decompress response bodies.
+// Node.js http passes raw bytes through, leaving Content-Encoding handling to the caller.
+export const testHttpClientGzipResponseNotAutoDecompressed = {
+  async test(_ctrl, env) {
+    const { promise, resolve, reject } = Promise.withResolvers();
+    http.get(
+      {
+        hostname: env.SIDECAR_HOSTNAME,
+        port: env.GZIP_SERVER_PORT,
+      },
+      (res) => {
+        strictEqual(res.headers['content-encoding'], 'gzip');
+        const chunks = [];
+        res.on('data', (chunk) => chunks.push(chunk));
+        res.on('end', () => {
+          try {
+            const raw = Buffer.concat(chunks);
+            // The body should still be compressed — manually gunzipping
+            // should recover the original text.
+            const decompressed = gunzipSync(raw);
+            strictEqual(decompressed.toString(), 'hello from gzip server');
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
+      }
+    );
+    await promise;
+  },
+};
+
 // Relevant Node.js tests
 // - [ ] test/parallel/test-http-client-abort-destroy.js
 // - [ ] test/parallel/test-http-client-abort-event.js

src/workerd/api/node/tests/http-client-nodejs-test.wd-test

@@ -15,6 +15,7 @@ const unitTests :Workerd.Config = (
           (name = "DEFAULT_HEADERS_EXIST_PORT", fromEnvironment = "DEFAULT_HEADERS_EXIST_PORT"),
           (name = "REQUEST_ARGUMENTS_PORT", fromEnvironment = "REQUEST_ARGUMENTS_PORT"),
           (name = "HELLO_WORLD_SERVER_PORT", fromEnvironment = "HELLO_WORLD_SERVER_PORT"),
+          (name = "GZIP_SERVER_PORT", fromEnvironment = "GZIP_SERVER_PORT"),
         ],
       )
     ),