Merge pull request #6294 from cloudflare/yagiz/fix-zlib-memory-lmit-exceeded

Author: anonrig

src/workerd/api/node/tests/zlib-nodejs-test.js

@@ -2857,3 +2857,57 @@ export const zlibParamsAfterWriteNoStalePointers = {
     );
   },
 };
+
+// Regression test for https://github.com/cloudflare/workerd/issues/6286
+// inflateRawSync throws "Memory limit exceeded" when maxOutputLength is set
+// to exactly the decompressed size. This is an off-by-one error in the
+// GrowableBuffer: after zlib fills the buffer completely (avail_out == 0),
+// the processing loop tries to add another chunk which exceeds maxCapacity.
+export const maxOutputLengthExactSize = {
+  test() {
+    // Create a known payload and compress it with deflateRaw
+    const original = Buffer.from('a]b]c]d]e]f]g]h]i]j]k]l]m]n]o]p]'.repeat(32));
+    const compressed = zlib.deflateRawSync(original);
+    const exactSize = original.length;
+
+    // This should succeed — maxOutputLength is exactly the output size.
+    // Before the fix, this throws RangeError: "Memory limit exceeded"
+    const decompressed = zlib.inflateRawSync(compressed, {
+      maxOutputLength: exactSize,
+    });
+    assert.deepStrictEqual(decompressed, original);
+
+    // Sanity check: maxOutputLength + 1 also works
+    const decompressed2 = zlib.inflateRawSync(compressed, {
+      maxOutputLength: exactSize + 1,
+    });
+    assert.deepStrictEqual(decompressed2, original);
+
+    // Same bug affects inflateSync
+    const compressedZlib = zlib.deflateSync(original);
+    const decompressed3 = zlib.inflateSync(compressedZlib, {
+      maxOutputLength: exactSize,
+    });
+    assert.deepStrictEqual(decompressed3, original);
+
+    // And gunzipSync
+    const compressedGzip = zlib.gzipSync(original);
+    const decompressed4 = zlib.gunzipSync(compressedGzip, {
+      maxOutputLength: exactSize,
+    });
+    assert.deepStrictEqual(decompressed4, original);
+
+    // And brotliDecompressSync
+    const compressedBrotli = zlib.brotliCompressSync(original);
+    const decompressed5 = zlib.brotliDecompressSync(compressedBrotli, {
+      maxOutputLength: exactSize,
+    });
+    assert.deepStrictEqual(decompressed5, original);
+
+    // Verify that a maxOutputLength that's genuinely too small still throws
+    assert.throws(
+      () => zlib.inflateRawSync(compressed, { maxOutputLength: exactSize - 1 }),
+      RangeError
+    );
+  },
+};

src/workerd/api/node/zlib-util.c++

@@ -93,6 +93,10 @@ class GrowableBuffer final {
     }
   }
 
+  bool atMaxCapacity() const {
+    return size() >= maxCapacity;
+  }
+
  private:
   kj::ArrayBuilder<kj::byte> builder;
   size_t chunkSize;
@@ -670,6 +674,8 @@ void BrotliEncoderContext::work() {
   lastResult = BrotliEncoderCompressStream(
       state.get(), flush, &availIn, &internalNext, &availOut, &nextOut, nullptr);
   nextIn += internalNext - nextIn;
+
+  streamEnd = lastResult && BrotliEncoderIsFinished(state.get());
 }
 
 kj::Maybe<CompressionError> BrotliEncoderContext::initialize(
@@ -709,6 +715,10 @@ kj::Maybe<CompressionError> BrotliEncoderContext::getError() const {
   return kj::none;
 }
 
+bool BrotliEncoderContext::isStreamEnd() const {
+  return streamEnd;
+}
+
 BrotliDecoderContext::BrotliDecoderContext(ZlibMode _mode): BrotliContext(_mode) {
   auto instance = BrotliDecoderCreateInstance(alloc_brotli, free_brotli, alloc_opaque_brotli);
   state = kj::disposeWith<BrotliDecoderDestroyInstance>(instance);
@@ -770,6 +780,10 @@ kj::Maybe<CompressionError> BrotliDecoderContext::getError() const {
   return kj::none;
 }
 
+bool BrotliDecoderContext::isStreamEnd() const {
+  return lastResult == BROTLI_DECODER_RESULT_SUCCESS;
+}
+
 // =======================================================================================
 // Zstd Implementation
 
@@ -892,6 +906,11 @@ kj::Maybe<CompressionError> ZstdEncoderContext::getError() const {
   return kj::none;
 }
 
+bool ZstdEncoderContext::isStreamEnd() const {
+  // ZSTD_compressStream2 returns 0 when flush_ == ZSTD_e_end and the frame is fully flushed.
+  return !ZSTD_isError(lastResult) && lastResult == 0;
+}
+
 ZstdDecoderContext::ZstdDecoderContext(ZlibMode _mode)
     : ZstdContext(_mode),
       dctx_(kj::disposeWith<zstdFreeDCtx>(ZSTD_createDCtx())) {}
@@ -959,6 +978,11 @@ kj::Maybe<CompressionError> ZstdDecoderContext::getError() const {
   return kj::none;
 }
 
+bool ZstdDecoderContext::isStreamEnd() const {
+  // ZSTD_decompressStream returns 0 when a frame is completely decoded and fully flushed.
+  return !ZSTD_isError(lastResult) && lastResult == 0;
+}
+
 template <typename CompressionContext>
 jsg::Ref<ZlibUtil::ZstdCompressionStream<CompressionContext>> ZlibUtil::ZstdCompressionStream<
     CompressionContext>::constructor(jsg::Lock& js, ZlibModeValue mode) {
@@ -1052,6 +1076,14 @@ static kj::Array<kj::byte> syncProcessBuffer(Context& ctx, GrowableBuffer& resul
     }
 
     result.adjustUnused(ctx.getAvailOut());
+
+    if (ctx.getAvailOut() == 0 && result.atMaxCapacity()) {
+      // The output buffer was completely filled and has reached maxOutputLength.
+      // If the stream is done, the output just happened to fit exactly — return it.
+      // Otherwise the decompressed data exceeds maxOutputLength.
+      JSG_REQUIRE(ctx.isStreamEnd(), RangeError, "Memory limit exceeded");
+      break;
+    }
   } while (ctx.getAvailOut() == 0);
 
   return result.releaseAsArray();

src/workerd/api/node/zlib-util.h

@@ -149,6 +149,12 @@ class ZlibContext final {
   // Ref: https://github.com/nodejs/node/blob/9edf4a0856681a7665bd9dcf2ca7cac252784b98/src/node_zlib.cc#L760
   void work();
 
+  // Returns true when the zlib stream has reached Z_STREAM_END, indicating
+  // that all compressed data has been fully processed.
+  bool isStreamEnd() const {
+    return err == Z_STREAM_END;
+  }
+
   uint getAvailIn() const {
     return stream.avail_in;
   };
@@ -284,9 +290,11 @@ class BrotliEncoderContext final: public BrotliContext {
   kj::Maybe<CompressionError> resetStream();
   kj::Maybe<CompressionError> setParams(int key, uint32_t value);
   kj::Maybe<CompressionError> getError() const;
+  bool isStreamEnd() const;
 
  private:
   bool lastResult = false;
+  bool streamEnd = false;
   kj::Own<BrotliEncoderStateStruct> state;
 };
 
@@ -304,6 +312,7 @@ class BrotliDecoderContext final: public BrotliContext {
   kj::Maybe<CompressionError> resetStream();
   kj::Maybe<CompressionError> setParams(int key, uint32_t value);
   kj::Maybe<CompressionError> getError() const;
+  bool isStreamEnd() const;
 
  private:
   BrotliDecoderResult lastResult = BROTLI_DECODER_RESULT_SUCCESS;
@@ -360,6 +369,7 @@ class ZstdEncoderContext final: public ZstdContext {
   kj::Maybe<CompressionError> resetStream();
   kj::Maybe<CompressionError> setParams(int key, int value);
   kj::Maybe<CompressionError> getError() const;
+  bool isStreamEnd() const;
 
  private:
   size_t lastResult = 0;
@@ -378,6 +388,7 @@ class ZstdDecoderContext final: public ZstdContext {
   kj::Maybe<CompressionError> resetStream();
   kj::Maybe<CompressionError> setParams(int key, int value);
   kj::Maybe<CompressionError> getError() const;
+  bool isStreamEnd() const;
 
  private:
   size_t lastResult = 0;