weftcodec 1.2.0__py3-none-any.whl

weftcodec/huffman.py

@@ -0,0 +1,212 @@
+"""`huffman-canonical-v1` decoder (WEFT-AI-CODEC §5.3.1).
+
+The serialized form is::
+
+    u8 code_lengths[256]   # 0 = symbol absent
+    || bitstream           # MSB-first, zero-padded to a byte boundary
+
+The decoder rebuilds the canonical table from the 256 lengths and rejects, in
+this order:
+
+1. a length greater than 24 -> ``NonCanonical``;
+2. an over-subscribed or incomplete prefix table (Kraft sum, 24-bit
+   resolution, not exactly ``2**24``) -> ``NonCanonical``, except the
+   single-symbol form (one used symbol with length 1);
+3. a truncated bitstream -> ``Corrupt``;
+4. extra trailing bytes, or padding bits that are not zero -> ``Corrupt``;
+5. an output length other than the declared ``element_count`` -> ``Corrupt``.
+
+``decode`` consumes the whole slice; ``decode_prefix`` consumes exactly the
+256-byte table plus this stream's own padded bitstream and returns that length
+so callers can concatenate independently coded streams (modes 11/12/13/14/15/
+16).
+
+This is a deliberately straightforward canonical decoder: performance is not
+the goal, exactness and identical error classes are.
+"""
+
+from __future__ import annotations
+
+from .errors import Corrupt, NonCanonical
+
+TABLE_BYTES = 256
+MAX_CODE_BITS = 24
+
+
+class _Canonical:
+    """Canonical tables for one 256-byte code-length header."""
+
+    __slots__ = ("counts", "first_code", "first_index", "symbols", "max_len")
+
+    def __init__(self) -> None:
+        self.counts = [0] * (MAX_CODE_BITS + 1)
+        self.first_code = [0] * (MAX_CODE_BITS + 1)
+        self.first_index = [0] * (MAX_CODE_BITS + 1)
+        self.symbols = []
+        self.max_len = 0
+
+
+def _build_table(lengths: bytes, expected_len: int) -> "tuple[_Canonical | None, bytes | None, int]":
+    """Return ``(tables, single_symbol_output, used)``.
+
+    ``tables`` is ``None`` for the empty and single-symbol forms. The second
+    element is the pre-filled output for a single-symbol table.
+    """
+    if len(lengths) < TABLE_BYTES:
+        raise Corrupt(
+            "huffman-canonical-v1 stream is %d bytes, shorter than its %d-byte "
+            "code-length table" % (len(lengths), TABLE_BYTES)
+        )
+    lengths = lengths[:TABLE_BYTES]
+    counts = [0] * (MAX_CODE_BITS + 1)
+    used = 0
+    max_len = 0
+    for symbol, length in enumerate(lengths):
+        if length > MAX_CODE_BITS:
+            raise NonCanonical(
+                "huffman-canonical-v1 symbol %d has code length %d > %d"
+                % (symbol, length, MAX_CODE_BITS)
+            )
+        if length:
+            used += 1
+            counts[length] += 1
+            if length > max_len:
+                max_len = length
+
+    if used == 0:
+        if expected_len != 0:
+            raise Corrupt(
+                "huffman-canonical-v1 table has no symbols but the descriptor "
+                "declares a non-empty plane"
+            )
+        return None, b"", 0
+
+    if used == 1:
+        if max_len != 1:
+            raise NonCanonical(
+                "huffman-canonical-v1 single-symbol table must use code length 1"
+            )
+        if expected_len == 0:
+            raise NonCanonical(
+                "huffman-canonical-v1 non-empty table for an empty plane"
+            )
+        symbol = next(index for index, length in enumerate(lengths) if length)
+        return None, bytes([symbol]) * expected_len, 1
+
+    if expected_len == 0:
+        raise NonCanonical(
+            "huffman-canonical-v1 non-empty table for an empty plane"
+        )
+
+    # Kraft equality proves a complete prefix code without building a table.
+    space = 1 << MAX_CODE_BITS
+    used_space = 0
+    for length in range(1, MAX_CODE_BITS + 1):
+        used_space += counts[length] << (MAX_CODE_BITS - length)
+        if used_space > space:
+            raise NonCanonical(
+                "huffman-canonical-v1 code lengths are over-subscribed"
+            )
+    if used_space != space:
+        raise NonCanonical("huffman-canonical-v1 code lengths are incomplete")
+
+    table = _Canonical()
+    table.counts = counts
+    table.max_len = max_len
+    # Canonical assignment: sort used symbols by (code length, symbol).
+    order = sorted(
+        (length, symbol) for symbol, length in enumerate(lengths) if length
+    )
+    code = 0
+    previous = 0
+    assigned = [0] * (MAX_CODE_BITS + 1)
+    symbols = []
+    for length, symbol in order:
+        code <<= length - previous
+        if assigned[length] == 0:
+            table.first_code[length] = code
+            table.first_index[length] = len(symbols)
+        assigned[length] += 1
+        symbols.append(symbol)
+        code += 1
+        previous = length
+    table.symbols = symbols
+    return table, None, used
+
+
+def _decode_prefix_tables(
+    table: _Canonical, stream: bytes, expected_len: int
+) -> "tuple[bytes, int]":
+    out = bytearray()
+    bit = 0
+    total_bits = len(stream) * 8
+    current_code = 0
+    current_len = 0
+    counts = table.counts
+    first_code = table.first_code
+    first_index = table.first_index
+    symbols = table.symbols
+    max_len = table.max_len
+
+    while len(out) < expected_len:
+        if bit >= total_bits:
+            raise Corrupt(
+                "huffman-canonical-v1 bitstream ended after %d of %d bytes"
+                % (len(out), expected_len)
+            )
+        byte = stream[bit >> 3]
+        current_code = (current_code << 1) | ((byte >> (7 - (bit & 7))) & 1)
+        current_len += 1
+        bit += 1
+
+        slot = current_len
+        if slot <= max_len:
+            first = first_code[slot]
+            if current_code >= first:
+                offset = current_code - first
+                if offset < counts[slot]:
+                    out.append(symbols[first_index[slot] + offset])
+                    current_code = 0
+                    current_len = 0
+                    continue
+        if current_len >= max_len:
+            raise Corrupt(
+                "huffman-canonical-v1 bitstream contains an invalid code"
+            )
+
+    consumed_stream = (bit + 7) // 8
+    used_bits_in_last = bit % 8
+    if used_bits_in_last:
+        last = stream[consumed_stream - 1]
+        mask = (1 << (8 - used_bits_in_last)) - 1
+        if last & mask:
+            raise Corrupt("huffman-canonical-v1 padding bits must be zero")
+    return bytes(out), TABLE_BYTES + consumed_stream
+
+
+def decode_prefix(encoded: bytes, expected_len: int):
+    """Decode one stream and return ``(bytes, consumed)``.
+
+    ``consumed`` is the 256-byte table plus this stream's own padded bitstream.
+    Bytes after that prefix belong to the caller.
+    """
+    if len(encoded) < TABLE_BYTES:
+        raise Corrupt(
+            "huffman-canonical-v1 stream is %d bytes, shorter than its %d-byte "
+            "code-length table" % (len(encoded), TABLE_BYTES)
+        )
+    table, single, _used = _build_table(encoded[:TABLE_BYTES], expected_len)
+    if table is None:
+        return single, TABLE_BYTES
+    return _decode_prefix_tables(table, encoded[TABLE_BYTES:], expected_len)
+
+
+def decode(encoded: bytes, expected_len: int) -> bytes:
+    """Decode exactly one complete stream; any trailing byte is ``Corrupt``."""
+    out, consumed = decode_prefix(encoded, expected_len)
+    if consumed != len(encoded):
+        raise Corrupt(
+            "huffman-canonical-v1 bitstream has trailing bytes after the "
+            "declared output"
+        )
+    return out

weftcodec/identity.py

@@ -0,0 +1,141 @@
+"""Identity helpers: domain-separated BLAKE3 and frame verification."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import asdict, dataclass
+
+from .dependency import Resolver
+from .frame import Frame, parse
+from .hashing import (
+    DOMAIN_CHUNK,
+    HASH_LEN,
+    content_root,
+    content_root_hex,
+    hash_domain,
+    hash_domain_hex,
+)
+from .limits import Limits
+from .modes import decode_frame
+
+__all__ = [
+    "DOMAIN_CHUNK",
+    "HASH_LEN",
+    "VerifyResult",
+    "content_root",
+    "content_root_hex",
+    "hash_domain",
+    "hash_domain_hex",
+    "verify_bytes",
+    "verify_frame",
+]
+
+
+@dataclass
+class VerifyResult:
+    """Structured result of parsing and decoding one frame."""
+
+    ok: bool
+    mode: "str | None" = None
+    mode_value: "int | None" = None
+    domain: "int | None" = None
+    flags: "list[str] | None" = None
+    features: "list[str] | None" = None
+    decoded_len: "int | None" = None
+    body_len: "int | None" = None
+    chunk_count: "int | None" = None
+    dependency_count: "int | None" = None
+    level: "int | None" = None
+    content_hash: "str | None" = None
+    computed_content_hash: "str | None" = None
+    error_class: "str | None" = None
+    error: "str | None" = None
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def _fill_from_header(result: VerifyResult, frame: Frame) -> None:
+    header = frame.header
+    result.mode = header.mode_name
+    result.mode_value = header.mode
+    result.domain = header.domain
+    result.flags = header.flag_names()
+    result.features = header.feature_names()
+    result.decoded_len = header.decoded_len
+    result.body_len = header.body_len
+    result.chunk_count = header.chunk_count
+    result.dependency_count = header.dep_count
+    result.level = header.level
+    result.content_hash = header.content_hash.hex()
+
+
+def verify_bytes(
+    data: bytes,
+    resolver: "Resolver | None" = None,
+    limits: "Limits | None" = None,
+    base_frame: "bytes | None" = None,
+) -> VerifyResult:
+    """Parse, decode, and verify every hash in ``data``."""
+    limits = limits or Limits()
+    result = VerifyResult(ok=False)
+    try:
+        frame = parse(data, limits)
+    except Exception as exc:  # typed Weft errors plus no unexpected swallowing
+        return _failure(result, exc)
+    _fill_from_header(result, frame)
+    try:
+        plaintext = decode_frame(frame, resolver, limits, base_frame)
+    except Exception as exc:
+        return _failure(result, exc, keep_header=True)
+    result.computed_content_hash = content_root_hex(plaintext)
+    result.ok = True
+    return result
+
+
+def verify_frame(
+    path: "str | os.PathLike[str]",
+    resolver: "Resolver | None" = None,
+    limits: "Limits | None" = None,
+    base_frame: "bytes | None" = None,
+) -> VerifyResult:
+    """Read ``path``, parse, decode, and verify it. Never raises for codec errors."""
+    limits = limits or Limits()
+    result = VerifyResult(ok=False)
+    try:
+        size = os.path.getsize(path)
+    except OSError as exc:
+        from .errors import Io
+
+        return _failure(result, Io("cannot stat %s: %s" % (path, exc)))
+    if size > limits.max_frame_bytes:
+        from .errors import LimitExceeded
+
+        return _failure(
+            result,
+            LimitExceeded(
+                "frame length %d exceeds limit %d" % (size, limits.max_frame_bytes)
+            ),
+        )
+    try:
+        with open(path, "rb") as handle:
+            data = handle.read()
+    except OSError as exc:
+        from .errors import Io
+
+        return _failure(result, Io("cannot read %s: %s" % (path, exc)))
+    return verify_bytes(data, resolver, limits, base_frame)
+
+
+def _failure(
+    result: VerifyResult, exc: Exception, keep_header: bool = False
+) -> VerifyResult:
+    error_class = getattr(exc, "error_class", None)
+    if error_class is None:
+        raise exc
+    result.ok = False
+    result.error_class = error_class
+    result.error = str(exc)
+    if not keep_header:
+        result.mode = None
+    return result

weftcodec/limits.py

@@ -0,0 +1,58 @@
+"""Decoder/encoder resource limits (WEFTCODEC-FORMAT §10).
+
+Defaults are the normative v1 values. Every field can be lowered; nothing in
+this package raises a limit above the frozen default.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, fields, replace
+
+DEFAULT_MAX_FRAME_BYTES = 1 << 30
+DEFAULT_MAX_DECODED_BYTES = 4 << 30
+DEFAULT_MAX_DEPENDENCY_DEPTH = 8
+DEFAULT_MAX_DEPENDENCY_COUNT = 64
+DEFAULT_MAX_TOTAL_FETCHED_BYTES = 8 << 30
+DEFAULT_MAX_CHUNKS = 1_048_576
+DEFAULT_MAX_DICTIONARY_BYTES = 64 << 20
+DEFAULT_MAX_STRUCTURED_DEPTH = 64
+DEFAULT_MAX_STRUCTURED_ITEMS = 1_000_000
+DEFAULT_MAX_STRING_BYTES = 256 << 20
+DEFAULT_MAX_DELTA_OPS = 16_777_216
+DEFAULT_MAX_TENSOR_ELEMENTS = 1 << 32
+DEFAULT_MAX_TENSOR_RANK = 8
+DEFAULT_MAX_TOKENS = 1 << 24
+DEFAULT_MAX_SPARSE_RUNS = 1 << 24
+
+
+@dataclass(frozen=True)
+class Limits:
+    max_frame_bytes: int = DEFAULT_MAX_FRAME_BYTES
+    max_decoded_bytes: int = DEFAULT_MAX_DECODED_BYTES
+    max_dependency_depth: int = DEFAULT_MAX_DEPENDENCY_DEPTH
+    max_dependency_count: int = DEFAULT_MAX_DEPENDENCY_COUNT
+    max_total_fetched_bytes: int = DEFAULT_MAX_TOTAL_FETCHED_BYTES
+    max_chunks: int = DEFAULT_MAX_CHUNKS
+    max_dictionary_bytes: int = DEFAULT_MAX_DICTIONARY_BYTES
+    max_structured_depth: int = DEFAULT_MAX_STRUCTURED_DEPTH
+    max_structured_items: int = DEFAULT_MAX_STRUCTURED_ITEMS
+    max_string_bytes: int = DEFAULT_MAX_STRING_BYTES
+    max_delta_ops: int = DEFAULT_MAX_DELTA_OPS
+    max_tensor_elements: int = DEFAULT_MAX_TENSOR_ELEMENTS
+    max_tensor_rank: int = DEFAULT_MAX_TENSOR_RANK
+    max_tokens: int = DEFAULT_MAX_TOKENS
+    max_sparse_runs: int = DEFAULT_MAX_SPARSE_RUNS
+
+    @classmethod
+    def from_overrides(cls, overrides: "dict[str, int] | None") -> "Limits":
+        """Build limits from a vector's ``limits`` object.
+
+        Unknown keys are rejected so a typo can never silently widen a bound.
+        """
+        if not overrides:
+            return cls()
+        known = {f.name for f in fields(cls)}
+        unknown = set(overrides) - known
+        if unknown:
+            raise ValueError("unknown limit(s): %s" % ", ".join(sorted(unknown)))
+        return replace(cls(), **{k: int(v) for k, v in overrides.items()})