spillgate 0.1.0__py3-none-any.whl

spillgate/__init__.py

@@ -0,0 +1,77 @@
+"""spillgate: decode untrusted compressed data under mandatory limits.
+
+A decompression bomb is a small input that expands to an enormous output. spillgate
+stops it at the gate: every decoding layer has a hard output limit, an expansion
+ratio limit and a bounded chain length, and anything that exceeds a limit ends in
+a typed error, never in silent truncation.
+
+>>> from spillgate import Limits, decode_all
+>>> import gzip
+>>> decode_all(gzip.compress(b"hello"), "gzip", Limits(max_output=1 << 20))
+b'hello'
+
+The normative rules are in ``spec/SPEC.md``; rule IDs such as ``SG-R-LAY-03`` in
+comments and docstrings point there. Design principle: when in doubt, refuse.
+"""
+
+from __future__ import annotations
+
+from .codec import Codec, CodecReader, Source
+from .decoder import Decoder, decode_all, reader
+from .errors import (
+    CanceledError,
+    ClosedError,
+    CorruptError,
+    InputLimitError,
+    InternalError,
+    InvalidLimitsError,
+    InvalidOptionError,
+    Kind,
+    LayerLimitError,
+    MalformedEncodingError,
+    MemoryLimitError,
+    OutputLimitError,
+    RatioLimitError,
+    SourceError,
+    SpillgateError,
+    UnsupportedEncodingError,
+)
+from .limits import (
+    DEFAULT_MAX_LAYERS,
+    DEFAULT_MAX_RATIO,
+    DEFAULT_RATIO_GRACE,
+    HARD_MAX_LAYERS,
+    HARD_MAX_RATIO,
+    Limits,
+)
+
+__all__ = [
+    "DEFAULT_MAX_LAYERS",
+    "DEFAULT_MAX_RATIO",
+    "DEFAULT_RATIO_GRACE",
+    "HARD_MAX_LAYERS",
+    "HARD_MAX_RATIO",
+    "CanceledError",
+    "ClosedError",
+    "Codec",
+    "CodecReader",
+    "CorruptError",
+    "Decoder",
+    "InputLimitError",
+    "InternalError",
+    "InvalidLimitsError",
+    "InvalidOptionError",
+    "Kind",
+    "LayerLimitError",
+    "Limits",
+    "MalformedEncodingError",
+    "MemoryLimitError",
+    "OutputLimitError",
+    "RatioLimitError",
+    "Source",
+    "SourceError",
+    "SpillgateError",
+    "UnsupportedEncodingError",
+    "decode_all",
+    "reader",
+]

spillgate/_cancel.py

@@ -0,0 +1,25 @@
+"""Polling the caller's ``cancel`` callable, with its failures contained."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from .errors import CanceledError, InternalError, SpillgateError
+
+
+def poll(cancel: Callable[[], bool] | None, layer: int) -> SpillgateError | None:
+    """Return the error to make sticky if the caller wants to stop, else ``None``.
+
+    A callable that raises is a bug in the caller's code, not a reason to let a foreign
+    exception escape half-way through a read: it becomes a sticky ``internal`` error
+    (SG-I-05). ``KeyboardInterrupt`` and ``SystemExit`` are not ``Exception`` and pass.
+    """
+    if cancel is None:
+        return None
+    try:
+        wanted = bool(cancel())
+    except Exception as error:
+        failure = InternalError(f"the cancel callable raised {type(error).__name__}", layer=layer)
+        failure.__cause__ = error
+        return failure
+    return CanceledError("canceled by the caller", layer=layer) if wanted else None

spillgate/_chain.py

@@ -0,0 +1,117 @@
+"""The reader returned to the caller: the last layer plus the end-to-end ratio check."""
+
+from __future__ import annotations
+
+import contextlib
+import io
+from typing import Any
+
+from ._guard import MAX_STEP, LayerGuard
+from ._meter import InputMeter
+from .errors import ClosedError, RatioLimitError, SpillgateError, fresh
+from .limits import EffectiveLimits
+
+_END = object()
+
+
+class ChainReader(io.RawIOBase):
+    """A binary stream of the decoded bytes. Use it like any other raw file object.
+
+    Reading can deliver bytes and then fail: treat any
+    :class:`~spillgate.SpillgateError` as failure of the whole body (SG-R-LAY-06).
+    Errors are sticky: every later read raises the same error (SG-R-LAY-05).
+    Closing does not close the caller's source. Not safe for concurrent use.
+    """
+
+    def __init__(
+        self,
+        top: LayerGuard,
+        source: InputMeter,
+        limits: EffectiveLimits,
+        codecs: list[Any],
+    ) -> None:
+        super().__init__()
+        self._top = top
+        self._source = source  # the meter of layer 1: what was taken from the caller's source
+        self._limits = limits
+        self._codecs = codecs  # codec readers that need releasing
+        self._produced = 0
+        self._done: object | SpillgateError | None = None
+
+    def readable(self) -> bool:
+        return True
+
+    def readinto(self, buffer: Any) -> int:
+        view = memoryview(buffer).cast("B")
+        if self.closed:
+            raise ClosedError("I/O operation on a closed spillgate reader")
+        if self._done is not None:
+            return self._replay()
+        if len(view) == 0:
+            return 0
+        try:
+            data = self._top.read(len(view))
+        except SpillgateError as error:
+            self._done = fresh(error)
+            raise
+        if not data:
+            self._done = _END
+            return 0
+        view[: len(data)] = data
+        self._produced += len(data)
+        # SG-R-RAT-03: the same ratio test, end to end. The error follows these bytes.
+        # A layer that already failed has the more specific error, so it wins.
+        if not self._top.failed and self._limits.ratio_exceeded(
+            self._produced, self._source.consumed
+        ):
+            self._done = RatioLimitError(
+                "end-to-end expansion over the whole chain", limit=self._limits.max_ratio
+            )
+        return len(data)
+
+    def read(self, size: int | None = -1) -> bytes:
+        """Read up to ``size`` bytes (at most one step); ``-1`` or ``None`` reads to the end.
+
+        The buffer is never larger than one step, whatever ``size`` is, so a huge
+        ``size`` does not allocate a huge buffer (SG-R-MEM-02).
+        """
+        if size is None or size < 0:
+            return self.readall()
+        buffer = bytearray(min(size, MAX_STEP))
+        count = self.readinto(buffer)
+        return bytes(buffer[:count])
+
+    def readall(self) -> bytes:
+        """Read to the end. Memory grows with the bytes produced, never with a limit."""
+        out = bytearray()
+        while True:
+            chunk = self.read(MAX_STEP)
+            if not chunk:
+                return bytes(out)
+            out += chunk
+
+    def _replay(self) -> int:
+        if self._done is _END:
+            return 0
+        assert isinstance(self._done, SpillgateError)
+        raise fresh(self._done)
+
+    def close(self) -> None:
+        """Release codec state. Idempotent; the caller's source is left open (SG-R-RDR-06)."""
+        if self.closed:
+            return
+        codecs, self._codecs = self._codecs, []
+        try:
+            for codec in codecs:
+                _close_contained(codec)
+        finally:
+            super().close()
+
+
+def _close_contained(codec: Any) -> None:
+    """Close a codec reader; a failing ``close`` must not stop the others (SG-R-CDC-08)."""
+    close = getattr(codec, "close", None)
+    if close is None:
+        return
+    with contextlib.suppress(Exception):  # contained on purpose
+        close()

spillgate/_deflate.py

@@ -0,0 +1,89 @@
+"""HTTP ``deflate`` (RFC 9110 section 8.4.1.2), SG-R-CDC-03.
+
+``deflate`` in HTTP means the zlib format (RFC 1950 around RFC 1951 data), but some
+senders omit the wrapper. The first two bytes decide: a valid zlib header means zlib,
+anything else is raw DEFLATE, unless ``strict_zlib`` forbids the fallback.
+"""
+
+from __future__ import annotations
+
+import zlib
+
+from ._inflate import RawInflater
+from ._meter import InputMeter
+from .errors import CorruptError
+
+_CM_DEFLATE = 8
+_MAX_CINFO = 7
+_FLAG_DICT = 1 << 5
+_CHECK_MODULUS = 31
+_TRAILER = 4  # Adler-32, big-endian
+
+
+def is_zlib_header(cmf: int, flg: int) -> bool:
+    """RFC 1950: CM = 8, CINFO <= 7 and (CMF * 256 + FLG) divisible by 31."""
+    return (
+        cmf & 0x0F == _CM_DEFLATE
+        and cmf >> 4 <= _MAX_CINFO
+        and (cmf << 8 | flg) % _CHECK_MODULUS == 0
+    )
+
+
+class DeflateCodec:
+    """The built-in ``deflate`` codec."""
+
+    def __init__(self, *, strict_zlib: bool = False) -> None:
+        self._strict = strict_zlib
+
+    def new_reader(self, source: InputMeter) -> DeflateReader:
+        return DeflateReader(source, strict=self._strict)
+
+
+class DeflateReader:
+    """Decodes one zlib or raw DEFLATE stream and checks that nothing follows it."""
+
+    def __init__(self, source: InputMeter, *, strict: bool) -> None:
+        self._source = source
+        self._strict = strict
+        self._body: RawInflater | None = None
+        self._adler: int | None = None  # None for raw DEFLATE
+        self._ended = False
+
+    def read(self, n: int) -> bytes:
+        if self._ended:
+            return b""
+        if self._body is None:
+            self._start()
+        assert self._body is not None
+        data = self._body.read(n)
+        if data:
+            if self._adler is not None:
+                self._adler = zlib.adler32(data, self._adler)
+            return data
+        self._finish()
+        self._ended = True
+        return b""
+
+    def close(self) -> None:
+        self._body = None
+
+    def _start(self) -> None:
+        header = self._source.take_exact(2, "deflate header")
+        if is_zlib_header(header[0], header[1]):
+            if header[1] & _FLAG_DICT:
+                raise CorruptError("deflate: preset dictionaries are not supported")
+            self._adler = 1  # the Adler-32 of no data
+        elif self._strict:
+            raise CorruptError("deflate: not a zlib stream")
+        else:
+            self._source.unread(header)  # raw DEFLATE: the two bytes are data
+        self._body = RawInflater(self._source, "deflate")
+
+    def _finish(self) -> None:
+        """The DEFLATE data ended: verify the Adler-32 and that nothing follows."""
+        if self._adler is not None:
+            trailer = self._source.take_exact(_TRAILER, "deflate trailer")
+            if int.from_bytes(trailer, "big") != self._adler:
+                raise CorruptError("deflate: checksum mismatch")
+        if not self._source.at_end():
+            raise CorruptError("deflate: unexpected data after the end of the stream")

spillgate/_guard.py

@@ -0,0 +1,154 @@
+"""The layer guard: everything the specification demands of one layer.
+
+Whatever the codec does, the guard never hands out more than ``max_output`` bytes
+in total (SG-I-01), never turns a limit hit into a clean end (SG-I-02), and keeps
+raising the same terminal condition once one occurred (SG-I-03).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from ._cancel import poll
+from ._meter import InputMeter
+from .errors import (
+    InternalError,
+    OutputLimitError,
+    RatioLimitError,
+    SpillgateError,
+    fresh,
+)
+from .limits import EffectiveLimits
+
+# The most a guard asks a codec for in one call. It bounds the memory of one step
+# (SG-R-MEM-02) whatever size of buffer the caller reads with; callers get short
+# reads, which every reader contract allows.
+MAX_STEP = 64 * 1024
+
+_END = object()  # sticky "clean end" marker, as opposed to an error
+
+
+class LayerGuard:
+    """Wraps the output of one decoding layer. Not safe for concurrent use."""
+
+    def __init__(
+        self,
+        codec: Any,
+        meter: InputMeter,
+        layer: int,
+        limits: EffectiveLimits,
+        cancel: Callable[[], bool] | None = None,
+    ) -> None:
+        self._codec = codec  # has read(n) -> bytes
+        self._cancel = cancel
+        self._meter = meter
+        self._layer = layer
+        self._limits = limits
+        self._produced = 0
+        self._done: object | SpillgateError | None = None  # None, _END or the error
+
+    @property
+    def failed(self) -> bool:
+        """True once the layer has a terminal error (a clean end is not a failure)."""
+        return isinstance(self._done, SpillgateError)
+
+    def read(self, n: int) -> bytes:
+        """Return up to ``n`` bytes; ``b""`` is the clean end.
+
+        Raises:
+            SpillgateError: the layer failed. The same error is raised on every
+                later call (SG-R-LAY-05).
+        """
+        if self._done is not None:
+            return self._replay()
+        if n <= 0:
+            return b""  # SG-R-RDR-02: a zero-length read changes nothing
+        stop = poll(self._cancel, self._layer)
+        if stop is not None:
+            self._done = stop
+            return self._replay()
+        remaining = self._limits.max_output - self._produced
+        if remaining <= 0:
+            return self._probe()
+        data = self._call_codec(min(n, remaining, MAX_STEP))
+        if not data:
+            self._done = _END
+            return b""
+        self._produced += len(data)
+        # The bytes that broke the ratio were produced first, so they are still
+        # delivered; the error follows on the next read (SG-R-LAY-07).
+        if self._limits.ratio_exceeded(self._produced, self._meter.consumed):
+            self._done = RatioLimitError(
+                f"output of {self._produced} bytes from {self._meter.consumed} input bytes",
+                layer=self._layer,
+                limit=self._limits.max_ratio,
+            )
+        return data
+
+    def _replay(self) -> bytes:
+        if self._done is _END:
+            return b""
+        assert isinstance(self._done, SpillgateError)
+        raise fresh(self._done)
+
+    def _probe(self) -> bytes:
+        """The budget is used up: is there one more byte? (SG-R-LAY-03)
+
+        If the codec yields a byte, the limit is exceeded and the byte is thrown
+        away. If the stream ends cleanly the output was exactly at the limit and
+        all is well. The codec's own validation, such as a trailer checksum, still
+        runs inside this call and can still report corruption.
+        """
+        data = self._call_codec(1)
+        if data:
+            self._done = OutputLimitError(
+                f"the layer would produce more than {self._limits.max_output} bytes",
+                layer=self._layer,
+                limit=self._limits.max_output,
+            )
+        else:
+            self._done = _END
+        return self._replay()
+
+    def _call_codec(self, n: int) -> bytes:
+        """One codec read, with its failures contained (SG-R-CDC-08)."""
+        try:
+            data = self._as_bytes(self._codec.read(n), n)
+        except SpillgateError as error:
+            failure = error.with_layer(self._layer)
+            self._done = fresh(failure)
+            try:
+                raise failure from failure.__cause__
+            finally:
+                del failure  # the traceback would otherwise keep this frame, and the guard, alive
+        except Exception as error:
+            failure = self._classify(error)
+            self._done = fresh(failure)
+            try:
+                raise failure from (failure.__cause__ or error)
+            finally:
+                del failure
+        return data
+
+    def _as_bytes(self, data: object, n: int) -> bytes:
+        """Check the codec's return value once, by its size in *bytes* (SG-R-CDC-08).
+
+        ``len()`` of a ``memoryview`` counts items, not bytes, so the size is taken from
+        ``nbytes`` of a byte view; the delivered value is a plain ``bytes`` copy.
+        """
+        if not isinstance(data, bytes | bytearray | memoryview):
+            raise InternalError(
+                f"codec returned {type(data).__name__}, not bytes", layer=self._layer
+            )
+        view = memoryview(data)
+        if view.nbytes > n:
+            raise InternalError("codec returned more bytes than requested", layer=self._layer)
+        return view.tobytes()
+
+    def _classify(self, error: Exception) -> SpillgateError:
+        """An unexpected exception is a codec bug, unless the source caused it."""
+        source_failure = self._meter.source_failure()
+        if source_failure is not None:
+            return source_failure.with_layer(self._layer)
+        return InternalError(f"codec raised {type(error).__name__}", layer=self._layer)

spillgate/_gzip.py

@@ -0,0 +1,128 @@
+"""gzip (RFC 1952), SG-R-CDC-02.
+
+The framing is parsed here, not by the ``gzip`` module, so every size is bounded
+and every rule of the specification is explicit: the DEFLATE body is decoded by
+:class:`~spillgate._inflate.RawInflater`.
+"""
+
+from __future__ import annotations
+
+import zlib
+
+from ._inflate import RawInflater
+from ._meter import InputMeter
+from .errors import CorruptError
+
+_ID1, _ID2, _CM_DEFLATE = 0x1F, 0x8B, 8
+_FLAG_HCRC, _FLAG_EXTRA, _FLAG_NAME, _FLAG_COMMENT = 1 << 1, 1 << 2, 1 << 3, 1 << 4
+_RESERVED_FLAGS = 0xE0  # RFC 1952 requires an error if any is set
+_FIXED_HEADER = 10
+_TRAILER = 8
+_MAX_STRING = 512  # FNAME and FCOMMENT, terminator included (SG-R-CDC-02)
+_MASK32 = 0xFFFFFFFF
+
+
+class GzipCodec:
+    """The built-in ``gzip`` codec."""
+
+    def new_reader(self, source: InputMeter) -> GzipReader:
+        return GzipReader(source)
+
+
+class GzipReader:
+    """Decodes a sequence of gzip members that share one output budget."""
+
+    def __init__(self, source: InputMeter) -> None:
+        self._source = source
+        self._body: RawInflater | None = None
+        self._crc = 0
+        self._size = 0  # modulo 2**32
+        self._pending_trailer = False  # the body ended; its trailer is checked on the next read
+
+    def read(self, n: int) -> bytes:
+        if self._body is None:
+            self._start_member(first=True)  # the first member is mandatory
+        while True:
+            if self._pending_trailer:
+                self._pending_trailer = False
+                if not self._finish_member():
+                    return b""  # clean end after the last member
+            assert self._body is not None
+            data = self._body.read(n)
+            if data:
+                self._crc = zlib.crc32(data, self._crc)
+                self._size = (self._size + len(data)) & _MASK32
+                return data
+            # The DEFLATE body ended. Its trailer is checked now, but bytes already
+            # produced were delivered first, as in the reference implementation.
+            self._pending_trailer = True
+
+    def close(self) -> None:
+        self._body = None
+
+    # -- members ---------------------------------------------------------------
+
+    def _finish_member(self) -> bool:
+        """Check the trailer; start the next member. False means a clean end."""
+        trailer = self._source.take_exact(_TRAILER, "gzip trailer")
+        if int.from_bytes(trailer[:4], "little") != self._crc:
+            raise CorruptError("gzip: checksum mismatch")
+        if int.from_bytes(trailer[4:], "little") != self._size:
+            raise CorruptError("gzip: size mismatch")  # ISIZE is checked, never trusted
+        self._crc = self._size = 0
+        return self._start_member(first=False)
+
+    def _start_member(self, *, first: bool) -> bool:
+        """Parse a member header. False means no further member (clean end)."""
+        if not first:
+            probe = self._source.take(1)
+            if not probe:
+                return False
+            self._source.unread(probe)
+        fixed = self._source.take_exact(_FIXED_HEADER, "gzip header")
+        if fixed[0] != _ID1 or fixed[1] != _ID2 or fixed[2] != _CM_DEFLATE:
+            raise CorruptError("gzip: invalid header")
+        flags = fixed[3]
+        if flags & _RESERVED_FLAGS:
+            raise CorruptError("gzip: reserved header flags are set")
+
+        header_crc = zlib.crc32(fixed)
+        if flags & _FLAG_EXTRA:
+            header_crc = self._skip_extra(header_crc)
+        if flags & _FLAG_NAME:
+            header_crc = self._skip_string(header_crc, "name")
+        if flags & _FLAG_COMMENT:
+            header_crc = self._skip_string(header_crc, "comment")
+        if flags & _FLAG_HCRC:
+            stored = int.from_bytes(self._source.take_exact(2, "gzip header checksum"), "little")
+            if stored != header_crc & 0xFFFF:
+                raise CorruptError("gzip: header checksum mismatch")
+
+        self._body = RawInflater(self._source, "gzip")
+        return True
+
+    def _skip_extra(self, header_crc: int) -> int:
+        """FEXTRA: at most 65535 bytes by format, read in small pieces (SG-R-MEM-01)."""
+        length_bytes = self._source.take_exact(2, "gzip extra field")
+        header_crc = zlib.crc32(length_bytes, header_crc)
+        remaining = int.from_bytes(length_bytes, "little")
+        while remaining > 0:
+            chunk = self._source.take_exact(min(remaining, _MAX_STRING), "gzip extra field")
+            header_crc = zlib.crc32(chunk, header_crc)
+            remaining -= len(chunk)
+        return header_crc
+
+    def _skip_string(self, header_crc: int, what: str) -> int:
+        """FNAME / FCOMMENT: zero-terminated, at most 512 bytes including the terminator."""
+        collected = bytearray()
+        while len(collected) < _MAX_STRING:
+            chunk = self._source.take(_MAX_STRING - len(collected))
+            if not chunk:
+                raise CorruptError(f"gzip: header {what} ends too early")
+            end = chunk.find(b"\0")
+            if end >= 0:
+                self._source.unread(chunk[end + 1 :])
+                collected += chunk[: end + 1]
+                return zlib.crc32(bytes(collected), header_crc)
+            collected += chunk
+        raise CorruptError(f"gzip: header {what} is longer than {_MAX_STRING - 1} bytes")

spillgate/_inflate.py

@@ -0,0 +1,71 @@
+"""A bounded raw-DEFLATE decoder on top of ``zlib.decompressobj``.
+
+The classic Python trap is ``decompressobj().decompress(data)`` without
+``max_length``: it expands the whole input at once. Here every call passes a
+``max_length`` greater than zero (``0`` would mean *unlimited*), and input the
+decoder did not use is handed back to the :class:`~spillgate._meter.InputMeter`, so
+the meter counts only what the decoder really consumed.
+"""
+
+from __future__ import annotations
+
+import zlib
+
+from ._meter import InputMeter
+from .errors import CorruptError
+
+_FEED = 4096  # compressed bytes offered per step; the rest is handed back
+
+
+class RawInflater:
+    """Decodes one raw DEFLATE stream (RFC 1951) from a meter.
+
+    ``read(n)`` returns 1..n bytes, or ``b""`` once the final block was decoded.
+    After that the bytes behind the stream are back in the meter, exactly where a
+    trailer (gzip, zlib) or the next member starts.
+    """
+
+    def __init__(self, source: InputMeter, what: str) -> None:
+        self._source = source
+        self._what = what
+        self._decoder = zlib.decompressobj(-zlib.MAX_WBITS)
+        self._may_have_output = False  # the last call filled its buffer completely
+
+    @property
+    def finished(self) -> bool:
+        return self._decoder.eof
+
+    def read(self, n: int) -> bytes:
+        """Return decoded bytes; ``b""`` means the stream has ended (its trailer is next).
+
+        Raises:
+            CorruptError: invalid DEFLATE data, or the input ends before the final block.
+        """
+        if n < 1:
+            raise ValueError("n must be at least 1")  # max_length 0 would mean unlimited
+        decoder = self._decoder
+        while not decoder.eof:
+            # A full buffer last time means output may still be pending inside zlib
+            # even though all input was used, so ask again before taking more input.
+            data = b"" if self._may_have_output else self._source.take(_FEED)
+            if not data and not self._may_have_output:
+                raise CorruptError(f"{self._what}: the stream ends too early")
+            out = self._decompress(data, n)
+            self._may_have_output = len(out) == n
+            if out:
+                return out
+        return b""
+
+    def _decompress(self, data: bytes, n: int) -> bytes:
+        decoder = self._decoder
+        try:
+            out = decoder.decompress(data, n)
+        except zlib.error as error:
+            raise CorruptError(f"{self._what}: invalid DEFLATE data ({error})") from None
+        # Bytes zlib has not used go back, so "consumed" stays exact: after the
+        # final block that is the trailer, before it the input held back by max_length.
+        leftover = decoder.unused_data if decoder.eof else decoder.unconsumed_tail
+        if leftover:
+            # zlib has not seen the unconsumed tail yet; it is taken from the meter again.
+            self._source.unread(bytes(leftover))
+        return out