halo-format 0.1.0__tar.gz

halo_format-0.1.0/.gitignore

@@ -0,0 +1,41 @@
+# Node / TypeScript
+node_modules/
+dist/
+*.tsbuildinfo
+.npm/
+pnpm-debug.log*
+npm-debug.log*
+
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+*.whl
+.venv/
+venv/
+.uv/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+
+# Coverage / test output
+coverage/
+.coverage
+htmlcov/
+
+# Release: Python distributions built in CI before the PyPI upload
+dist-python/
+
+# Editor / OS
+.DS_Store
+.idea/
+*.swp
+
+# Local env
+.env
+.env.local
+
+# Internal design docs — never commit
+private/

halo_format-0.1.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: halo-format
+Version: 0.1.0
+Summary: Halo core: encode a value into a content-addressed navigable tree and read it back, verified.
+Project-URL: Homepage, https://github.com/halo-format/halo
+License: MIT
+Requires-Python: >=3.10
+Requires-Dist: rfc8785>=0.1.2
+Description-Content-Type: text/markdown
+
+# halo-format
+
+Python port of the Halo core. Same interop contract as the TypeScript reference implementation:
+identical input produces identical handles in both ports, guaranteed by the shared conformance
+vectors. See the repo root `README.md` and `private/` for the design.

halo_format-0.1.0/README.md

@@ -0,0 +1,5 @@
+# halo-format
+
+Python port of the Halo core. Same interop contract as the TypeScript reference implementation:
+identical input produces identical handles in both ports, guaranteed by the shared conformance
+vectors. See the repo root `README.md` and `private/` for the design.

halo_format-0.1.0/pyproject.toml

@@ -0,0 +1,20 @@
+[project]
+name = "halo-format"
+version = "0.1.0"
+description = "Halo core: encode a value into a content-addressed navigable tree and read it back, verified."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+# Core has zero runtime deps beyond stdlib hashlib (sha256) and a vetted JCS canonicalizer
+# (rfc8785), which is the interop oracle for canonical bytes.
+dependencies = ["rfc8785>=0.1.2"]
+
+[project.urls]
+Homepage = "https://github.com/halo-format/halo"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/halo_format"]

halo_format-0.1.0/src/halo_format/__init__.py

@@ -0,0 +1,70 @@
+"""Public surface of halo_format.
+
+Producer:  encode, extend, merge
+Consumer:  open_ -> Navigator(walk, fetch, fetch_many, root); free walk/fetch/fetch_many
+Stores:    MemoryStore, FileStore
+Errors:    UnknownHandle, HashMismatch, WrongKind, CanonicalizationError, StoreError
+
+Re-exports only; implementation lives in the sibling modules.
+"""
+
+from .canonical import canonical, canonical_bytes
+from .errors import (
+    CanonicalizationError,
+    HaloError,
+    HashMismatch,
+    StoreError,
+    UnknownHandle,
+    WrongKind,
+)
+from .carve import auto_carve, make_auto_carve
+from .encode import encode, extend, merge
+from .envelope import build_envelope, verify_envelope
+from .hash import handle_of, hash_bytes
+from .navigate import Navigator, fetch, fetch_many, open_, walk
+from .node import (
+    branch_node,
+    decode,
+    is_branch,
+    is_leaf,
+    leaf_node,
+    node_handle,
+    serialize,
+)
+from .store import FileStore, MemoryStore
+from .summarize import derive_summary
+
+__all__ = [
+    "canonical",
+    "canonical_bytes",
+    "hash_bytes",
+    "handle_of",
+    "branch_node",
+    "leaf_node",
+    "is_branch",
+    "is_leaf",
+    "serialize",
+    "node_handle",
+    "decode",
+    "auto_carve",
+    "make_auto_carve",
+    "derive_summary",
+    "build_envelope",
+    "verify_envelope",
+    "encode",
+    "extend",
+    "merge",
+    "open_",
+    "walk",
+    "fetch",
+    "fetch_many",
+    "Navigator",
+    "MemoryStore",
+    "FileStore",
+    "HaloError",
+    "UnknownHandle",
+    "HashMismatch",
+    "WrongKind",
+    "CanonicalizationError",
+    "StoreError",
+]

halo_format-0.1.0/src/halo_format/canonical.py

@@ -0,0 +1,46 @@
+"""value -> canonical bytes (RFC 8785 JCS).
+
+The interop crux and a leaf dependency. Keys sorted by UTF-16 code unit, JCS string escaping,
+ECMAScript Number-to-String for numbers. v1 allows non-integer floats and leans on a vetted JCS
+library here. Must be byte-identical to the TypeScript port; defended by
+conformance/vectors/canonical.
+"""
+
+import math
+
+import rfc8785
+
+from .errors import CanonicalizationError
+
+
+def _assert_canonicalizable(value):
+    """Reject non-finite floats up front so the error matches across ports.
+
+    bool is a subclass of int in Python; rfc8785 handles it correctly, and we never treat a bool
+    as a number here, so no special-casing is needed.
+    """
+    if isinstance(value, float):
+        if not math.isfinite(value):
+            raise CanonicalizationError(f"non-finite number cannot be canonicalized: {value!r}")
+    elif isinstance(value, dict):
+        for v in value.values():
+            _assert_canonicalizable(v)
+    elif isinstance(value, (list, tuple)):
+        for v in value:
+            _assert_canonicalizable(v)
+
+
+def canonical_bytes(value) -> bytes:
+    """Canonical RFC 8785 (JCS) bytes (UTF-8) of a JSON value — the exact bytes that get hashed."""
+    _assert_canonicalizable(value)
+    try:
+        return rfc8785.dumps(value)
+    except CanonicalizationError:
+        raise
+    except Exception as e:  # rfc8785 raises its own/ValueError for unrepresentable input
+        raise CanonicalizationError(str(e)) from e
+
+
+def canonical(value) -> str:
+    """Canonical RFC 8785 (JCS) string form of a JSON value."""
+    return canonical_bytes(value).decode("utf-8")

halo_format-0.1.0/src/halo_format/carve.py

@@ -0,0 +1,41 @@
+"""carve(value, path) -> {"as": "leaf"} | {"as": "branch", "children": {...}, "summary"?: str}.
+
+auto_carve default: split a non-empty object into one branch per key; chunk arrays longer than a
+threshold (default 25) into contiguous slices; inline everything below a small byte threshold as a
+leaf. Hosts/skills override with task knowledge — an explicit carve always wins over the default.
+Bounds node size so no single node need be the whole payload. Pure and deterministic: the carve
+shape is part of what gets hashed.
+"""
+
+from .canonical import canonical_bytes
+
+DEFAULT_ARRAY_THRESHOLD = 25
+DEFAULT_INLINE_BYTES = 1024
+
+
+def make_auto_carve(array_threshold=DEFAULT_ARRAY_THRESHOLD, inline_bytes=DEFAULT_INLINE_BYTES):
+    """Build an auto_carve policy with custom thresholds. `auto_carve` is this with the defaults."""
+
+    def auto_carve(value, path):
+        # bool is a dict/list-free primitive in Python, so it falls through to leaf below.
+        if isinstance(value, dict):
+            if not value:
+                return {"as": "leaf"}  # {} is a terminal leaf, not a branch
+            if len(canonical_bytes(value)) <= inline_bytes:
+                return {"as": "leaf"}  # small object inlines whole
+            return {"as": "branch", "children": dict(value)}
+        if isinstance(value, list):
+            if len(value) <= array_threshold:
+                return {"as": "leaf"}  # short array inlines as one leaf
+            children = {}
+            chunks = (len(value) + array_threshold - 1) // array_threshold
+            for i in range(chunks):
+                children[str(i)] = value[i * array_threshold : (i + 1) * array_threshold]
+            return {"as": "branch", "children": children}  # each chunk re-carves to a leaf
+        return {"as": "leaf"}  # primitives (None, bool, number, str) are always leaves
+
+    return auto_carve
+
+
+# The default carve policy: object-by-key, chunk long arrays, inline small subtrees.
+auto_carve = make_auto_carve()

halo_format-0.1.0/src/halo_format/encode.py

@@ -0,0 +1,76 @@
+"""Bottom-up Merkle build, plus extend / merge.
+
+_build_node recurses children-first so a parent references its children by handle, which gives the
+tree its Merkle structure and free dedup (store.put is idempotent and content-keyed). encode returns
+{"handle", "envelope"}. extend(root, name, value) folds a new branch onto an existing root (last
+write wins on name collision); merge(roots) combines several roots' branch tables. Both reuse every
+unchanged child handle, so the only new node is the root, and because nodes are immutable the prior
+root stays in the store — version history and a change audit for free.
+
+Sync to match the Python Store (see store.py's note on the async divergence from TypeScript).
+"""
+
+from .carve import auto_carve
+from .envelope import build_envelope
+from .errors import WrongKind
+from .node import branch_node, decode, is_branch, leaf_node, serialize
+from .summarize import derive_summary
+
+
+def _build_node(value, path, store, carve, summarize):
+    """Build one node and everything beneath it, returning the node's handle."""
+    decision = carve(value, path)
+    if decision["as"] == "leaf":
+        node = leaf_node(value)
+    else:
+        branches = {}
+        for name, child in decision["children"].items():
+            branches[name] = _build_node(child, path + [name], store, carve, summarize)
+        summary = decision.get("summary")
+        if summary is None:
+            summary = summarize(value, branches)
+        node = branch_node(summary, branches)
+    return store.put(serialize(node))
+
+
+def _result_for(root, store, alg):
+    """Read the stored root node back and wrap it in an envelope."""
+    root_node = decode(store.get(root))
+    return {"handle": root, "envelope": build_envelope(root, root_node, alg)}
+
+
+def encode(value, store, *, carve=None, summarize=None, alg="sha256"):
+    """Encode a JSON value into content-addressed nodes and return its halo."""
+    carve = carve or auto_carve
+    summarize = summarize or derive_summary
+    root = _build_node(value, [], store, carve, summarize)
+    return _result_for(root, store, alg)
+
+
+def _read_branch(root, store):
+    node = decode(store.get(root))
+    if not is_branch(node):
+        raise WrongKind("extend/merge expect a branch root")
+    return node
+
+
+def extend(root, name, value, store, *, carve=None, summarize=None, alg="sha256"):
+    """Fold `value` onto `root` as a new branch named `name` (last write wins on collision)."""
+    carve = carve or auto_carve
+    summarize = summarize or derive_summary
+    base = _read_branch(root, store)
+    child_handle = _build_node(value, [name], store, carve, summarize)
+    branches = {**base["branches"], name: child_handle}
+    new_root = store.put(serialize(branch_node(summarize(value, branches), branches)))
+    return _result_for(new_root, store, alg)
+
+
+def merge(roots, store, *, summarize=None, alg="sha256"):
+    """Combine several roots' branch tables into one new root (last write wins, in order)."""
+    summarize = summarize or derive_summary
+    branches = {}
+    for root in roots:
+        node = _read_branch(root, store)
+        branches = {**branches, **node["branches"]}
+    new_root = store.put(serialize(branch_node(summarize(None, branches), branches)))
+    return _result_for(new_root, store, alg)

halo_format-0.1.0/src/halo_format/envelope.py

@@ -0,0 +1,47 @@
+"""Build the root envelope — the only thing that crosses into the model's context.
+
+    {"halo": "1", "alg": ..., "root": ..., "view": {"summary", "branches"}, "source"?: ...}
+
+view inlines the root node's content so a consumer navigates with zero fetches and can verify by
+re-hashing the reconstructed root against root. source is envelope-only identification metadata
+(id, tool, args, ts) and is NEVER hashed. (Full envelope verification and source population land
+with the navigator/adapter; this module is the builder the encode pipeline returns.)
+"""
+
+from .node import branch_node, is_branch, node_handle
+
+
+def _leaf_summary(value):
+    """A leaf root has no branches; describe its shape instead. Structural + cross-port identical."""
+    if value is None:
+        return "leaf: null"
+    if isinstance(value, bool):  # before int/float: bool is an int subclass in Python
+        return "leaf: boolean"
+    if isinstance(value, (int, float)):
+        return "leaf: number"
+    if isinstance(value, str):
+        return "leaf: string"
+    if isinstance(value, list):
+        return f"leaf: array of {len(value)} items"
+    return f"leaf: object with {len(value)} keys"
+
+
+def build_envelope(root, root_node, alg="sha256"):
+    """Build the envelope for an already-stored root node."""
+    if is_branch(root_node):
+        view = {"summary": root_node["summary"], "branches": dict(root_node["branches"])}
+    else:
+        view = {"summary": _leaf_summary(root_node["value"]), "branches": {}}
+    return {"halo": "1", "alg": alg, "root": root, "view": view}
+
+
+def verify_envelope(envelope):
+    """Self-verify an envelope from its inlined view alone (no store): re-hash the reconstructed
+    root branch node and compare to the claimed root. source is excluded by construction — it is
+    never part of a node — so two envelopes differing only in source verify identically.
+
+    Only a BRANCH root can be self-verified this way; a leaf root's value is not inlined in the view,
+    so this returns False for leaf roots and the caller must verify against the store (open_/fetch).
+    """
+    reconstructed = branch_node(envelope["view"]["summary"], envelope["view"]["branches"])
+    return node_handle(reconstructed, envelope["alg"]) == envelope["root"]

halo_format-0.1.0/src/halo_format/errors.py

@@ -0,0 +1,29 @@
+"""Typed errors with shared meaning across ports.
+
+UnknownHandle, HashMismatch (the tamper signal, never swallowed), WrongKind,
+CanonicalizationError, StoreError.
+"""
+
+
+class HaloError(Exception):
+    """Base class for all Halo errors."""
+
+
+class UnknownHandle(HaloError):
+    """A handle is absent from the store."""
+
+
+class HashMismatch(HaloError):
+    """Read bytes do not verify against the requested handle — the tamper signal."""
+
+
+class WrongKind(HaloError):
+    """walk was called on a leaf, or fetch on a branch."""
+
+
+class CanonicalizationError(HaloError):
+    """A value cannot be canonicalized (e.g. a non-finite number)."""
+
+
+class StoreError(HaloError):
+    """Wraps an adapter-level store failure."""

halo_format-0.1.0/src/halo_format/hash.py

@@ -0,0 +1,29 @@
+"""canonical bytes -> handle.
+
+handle = "h:" + sha256(canonical(node)).hexdigest(), full 64-hex. Algorithm pluggable behind a
+small registry keyed by alg (default sha256); the envelope declares it per-tree. Uses stdlib
+hashlib. Leaf dependency, byte-exact across ports.
+"""
+
+import hashlib
+
+from .canonical import canonical_bytes
+
+# Algorithm registry keyed by alg. A future move (e.g. blake3) registers a function here and the
+# envelope's alg selects it; call sites never hard-code the algorithm.
+_REGISTRY = {
+    "sha256": lambda b: hashlib.sha256(b).hexdigest(),
+}
+
+
+def hash_bytes(data: bytes, alg: str = "sha256") -> str:
+    """Hash canonical bytes into a handle: "h:" + lowercase hex digest under alg."""
+    fn = _REGISTRY.get(alg)
+    if fn is None:
+        raise ValueError(f"unknown hash alg: {alg}")
+    return "h:" + fn(data)
+
+
+def handle_of(value, alg: str = "sha256") -> str:
+    """Convenience: handle of a JSON value = hash_bytes(canonical_bytes(value))."""
+    return hash_bytes(canonical_bytes(value), alg)

halo_format-0.1.0/src/halo_format/navigate.py

@@ -0,0 +1,169 @@
+"""open_(envelope, store) -> Navigator(walk, fetch, fetch_many, root), plus the free
+walk/fetch/fetch_many low-level path.
+
+Every read verifies: read bytes, recompute the hash under the bound alg, compare to the requested
+handle, and only then decode. That single check is what makes the store untrusted — substituted or
+corrupted bytes fail at the moment they are read, before their contents are used, and the guarantee
+propagates from the verified root down every branch to every leaf (the trust chain).
+
+walk returns a branch's summary + child handles (never leaf data); fetch returns one verified leaf;
+fetch_many verifies several in one call with per-ref results, so one unknown or tampered entry never
+sinks the batch. The Navigator additionally resolves map-scoped refs (e.g. m1.income, or income /
+income.monthly against the opened envelope) to handles before verifying; raw handles always work and
+need no registration.
+
+Sync to match the Python Store (see store.py's note on the async divergence from TypeScript).
+"""
+
+from .envelope import verify_envelope
+from .errors import HaloError, HashMismatch, UnknownHandle, WrongKind
+from .hash import hash_bytes
+from .node import decode, is_branch, is_leaf
+
+
+def _is_raw_handle(ref):
+    return ref.startswith("h:")
+
+
+def _read_verified(handle, store, alg):
+    """Read a handle's bytes and verify them against it before decoding. The load-bearing check."""
+    data = store.get(handle)  # raises UnknownHandle if absent
+    if hash_bytes(data, alg) != handle:
+        raise HashMismatch(handle)
+    return decode(data)
+
+
+def _get_many_bytes(store, handles):
+    get_many = getattr(store, "get_many", None)
+    if get_many is not None:
+        return get_many(handles)
+    out = []
+    for h in handles:
+        try:
+            out.append(store.get(h))
+        except UnknownHandle:
+            out.append(None)
+    return out
+
+
+def walk(handle, store, alg="sha256"):
+    """Verified walk of a branch handle: returns its summary and child handles, never leaf data."""
+    node = _read_verified(handle, store, alg)
+    if not is_branch(node):
+        raise WrongKind(f"walk expects a branch: {handle}")
+    return {"summary": node["summary"], "branches": node["branches"]}
+
+
+def fetch(handle, store, alg="sha256"):
+    """Verified fetch of a leaf handle: returns its value."""
+    node = _read_verified(handle, store, alg)
+    if not is_leaf(node):
+        raise WrongKind(f"fetch expects a leaf: {handle}")
+    return node["value"]
+
+
+def _verify_batch(present, store, alg):
+    """Verify already-resolved (ref, handle) pairs independently. One backend round trip via
+    get_many. Missing -> UnknownHandle, bad hash -> HashMismatch, branch-where-leaf -> WrongKind."""
+    out = {}
+    byteses = _get_many_bytes(store, [h for _, h in present])
+    for (ref, handle), data in zip(present, byteses):
+        if data is None:
+            out[ref] = {"ok": False, "error": "UnknownHandle"}
+        elif hash_bytes(data, alg) != handle:
+            out[ref] = {"ok": False, "error": "HashMismatch"}
+        else:
+            node = decode(data)
+            out[ref] = (
+                {"ok": True, "value": node["value"]}
+                if is_leaf(node)
+                else {"ok": False, "error": "WrongKind"}
+            )
+    return out
+
+
+def fetch_many(refs, store, alg="sha256"):
+    """Verified batch fetch over raw handles (low-level path; each ref is treated as a handle)."""
+    return _verify_batch([(r, r) for r in refs], store, alg)
+
+
+class Navigator:
+    """Bound to an envelope's algorithm, store, and root. Resolves map-scoped refs and verifies
+    every read. Register additional envelopes (by their source.id) to resolve refs across several
+    maps in one session, e.g. m1.income vs m2.income."""
+
+    def __init__(self, primary, store):
+        self._primary = primary
+        self._store = store
+        self._alg = primary["alg"]
+        self.root = primary["root"]
+        self._registry = {}
+        self._register(primary)
+
+    def _register(self, env):
+        src = env.get("source")
+        if src and src.get("id"):
+            self._registry[src["id"]] = env
+
+    def register(self, env):
+        """Register another envelope so its mapId.branch refs resolve in this session."""
+        self._register(env)
+        return self
+
+    def resolve(self, ref):
+        """Resolve a ref (raw handle, mapId.branch[.child...], or a path on the opened envelope)."""
+        if _is_raw_handle(ref):
+            return ref
+        parts = ref.split(".")
+        if len(parts) >= 2 and parts[0] in self._registry:
+            env = self._registry[parts[0]]
+            return self._resolve_path(env["view"]["branches"], parts[1:], ref)
+        return self._resolve_path(self._primary["view"]["branches"], parts, ref)
+
+    def _resolve_path(self, branches, segs, ref):
+        if not segs:
+            raise UnknownHandle(f"empty ref: {ref}")
+        handle = branches.get(segs[0])
+        if handle is None:
+            raise UnknownHandle(f"unknown ref: {ref}")
+        for seg in segs[1:]:
+            node = _read_verified(handle, self._store, self._alg)
+            if not is_branch(node):
+                raise WrongKind(f"ref descends into a leaf: {ref}")
+            handle = node["branches"].get(seg)
+            if handle is None:
+                raise UnknownHandle(f"unknown ref: {ref}")
+        return handle
+
+    def walk(self, ref):
+        return walk(self.resolve(ref), self._store, self._alg)
+
+    def fetch(self, ref):
+        return fetch(self.resolve(ref), self._store, self._alg)
+
+    def fetch_many(self, refs):
+        present = []
+        out = {}
+        for r in refs:
+            try:
+                present.append((r, self.resolve(r)))
+            except HaloError as e:
+                # A resolution failure is surfaced per-entry (by error class name), never raised.
+                out[r] = {"ok": False, "error": type(e).__name__}
+        out.update(_verify_batch(present, self._store, self._alg))
+        return out
+
+
+def open_(envelope, store):
+    """Open an envelope into a Navigator, verifying the root. Zero fetches for an honest branch root."""
+    alg = envelope["alg"]
+    # The inlined view lets us verify a branch root with no store read (verify_envelope re-hashes
+    # the reconstructed root node and compares to the claimed root handle).
+    if not verify_envelope(envelope):
+        # Either a leaf root (value not inlined) or a tampered branch view. Read the real root to
+        # tell them apart: a branch here means the view was tampered.
+        node = _read_verified(envelope["root"], store, alg)
+        if is_branch(node):
+            raise HashMismatch(f"envelope view does not match root: {envelope['root']}")
+        # leaf root: nothing branch-navigable; a later fetch(root) re-verifies it.
+    return Navigator(envelope, store)

halo_format-0.1.0/src/halo_format/node.py

@@ -0,0 +1,72 @@
+"""The node model and its (de)serialization.
+
+    BranchNode = {"k": "b", "summary": str, "branches": {name: Handle}}
+    LeafNode   = {"k": "l", "value": JsonValue}
+
+The k kind tag is part of the hashed content so a leaf and branch can never collide. Stored form
+is canonical(node) bytes, not the object.
+"""
+
+import json
+
+from .canonical import canonical_bytes
+from .errors import HaloError
+from .hash import hash_bytes
+
+
+def branch_node(summary: str, branches: dict) -> dict:
+    """Build a branch node. A shallow copy keeps it independent of the caller's map."""
+    return {"k": "b", "summary": summary, "branches": dict(branches)}
+
+
+def leaf_node(value) -> dict:
+    return {"k": "l", "value": value}
+
+
+def is_branch(node: dict) -> bool:
+    return node.get("k") == "b"
+
+
+def is_leaf(node: dict) -> bool:
+    return node.get("k") == "l"
+
+
+def serialize(node: dict) -> bytes:
+    """The exact bytes that get hashed and stored for a node: canonical(node)."""
+    return canonical_bytes(node)
+
+
+def node_handle(node: dict, alg: str = "sha256") -> str:
+    """A node's handle: hash over its canonical bytes. The Merkle building block."""
+    return hash_bytes(serialize(node), alg)
+
+
+def decode(data: bytes) -> dict:
+    """Decode stored bytes back into a validated node.
+
+    In normal use the bytes were verified against a handle first (navigate verifies on read), so
+    decode of verified bytes always succeeds; the validation here guards malformed input, it is
+    not a security boundary.
+    """
+    try:
+        obj = json.loads(data.decode("utf-8"))
+    except (ValueError, UnicodeDecodeError) as e:
+        raise HaloError(f"node bytes are not valid JSON: {e}") from e
+    if not isinstance(obj, dict):
+        raise HaloError("node must be a JSON object")
+    kind = obj.get("k")
+    if kind == "l":
+        if "value" not in obj:
+            raise HaloError("leaf node missing `value`")
+        return {"k": "l", "value": obj["value"]}
+    if kind == "b":
+        if not isinstance(obj.get("summary"), str):
+            raise HaloError("branch node `summary` must be a string")
+        branches = obj.get("branches")
+        if not isinstance(branches, dict):
+            raise HaloError("branch node `branches` must be an object")
+        for name, h in branches.items():
+            if not isinstance(h, str):
+                raise HaloError(f'branch handle for "{name}" must be a string')
+        return {"k": "b", "summary": obj["summary"], "branches": branches}
+    raise HaloError(f"unknown node kind: {kind!r}")

halo_format-0.1.0/src/halo_format/store.py

@@ -0,0 +1,97 @@
+"""The Store protocol plus MemoryStore and FileStore.
+
+    put(bytes) -> Handle   (content-addressed, idempotent, stores once)
+    get(handle) -> bytes   (raises UnknownHandle if absent)
+    has(handle) -> bool
+    get_many(handles) -> list[bytes | None]   (optional multi-get)
+
+Untrusted by design: every read verifies, so a buggy/hostile store cannot substitute data
+undetected. MemoryStore = in-process dict; FileStore = one file per handle under a dir.
+
+Note on async: the TypeScript Store is async (Promise-returning) per the SDK contract. The Python
+port is synchronous, which is idiomatic for the in-process and filesystem stores; a heavy/remote
+Python adapter can expose an async variant. The navigator built on top stays sync to match.
+"""
+
+import os
+
+from .errors import StoreError, UnknownHandle
+from .hash import hash_bytes
+
+
+def _hex_of(handle: str) -> str:
+    """A handle is "h:" + hex; the hex alone is the on-disk filename for FileStore."""
+    return handle[2:] if handle.startswith("h:") else handle
+
+
+class MemoryStore:
+    """In-process content-addressed store — the light deployment and the default for tests."""
+
+    def __init__(self, alg: str = "sha256"):
+        self._map: dict[str, bytes] = {}
+        self._alg = alg
+
+    def put(self, data: bytes) -> str:
+        handle = hash_bytes(data, self._alg)
+        # Idempotent + immutable: first writer wins, identical content is a no-op.
+        self._map.setdefault(handle, data)
+        return handle
+
+    def get(self, handle: str) -> bytes:
+        try:
+            return self._map[handle]
+        except KeyError:
+            raise UnknownHandle(handle) from None
+
+    def has(self, handle: str) -> bool:
+        return handle in self._map
+
+    def get_many(self, handles: list[str]) -> list[bytes | None]:
+        return [self._map.get(h) for h in handles]
+
+    def __len__(self) -> int:
+        return len(self._map)
+
+
+class FileStore:
+    """One file per handle under a directory — cross-step persistence inside a sandbox."""
+
+    def __init__(self, directory: str, alg: str = "sha256"):
+        self._dir = directory
+        self._alg = alg
+        try:
+            os.makedirs(directory, exist_ok=True)
+        except OSError as e:
+            raise StoreError(f"could not create store dir {directory}: {e}") from e
+
+    def _path_for(self, handle: str) -> str:
+        return os.path.join(self._dir, _hex_of(handle))
+
+    def put(self, data: bytes) -> str:
+        handle = hash_bytes(data, self._alg)
+        path = self._path_for(handle)
+        if not os.path.exists(path):  # immutable: never overwrite
+            with open(path, "wb") as f:
+                f.write(data)
+        return handle
+
+    def get(self, handle: str) -> bytes:
+        try:
+            with open(self._path_for(handle), "rb") as f:
+                return f.read()
+        except FileNotFoundError:
+            raise UnknownHandle(handle) from None
+        except OSError as e:
+            raise StoreError(f"could not read {handle}: {e}") from e
+
+    def has(self, handle: str) -> bool:
+        return os.path.exists(self._path_for(handle))
+
+    def get_many(self, handles: list[str]) -> list[bytes | None]:
+        out: list[bytes | None] = []
+        for h in handles:
+            try:
+                out.append(self.get(h))
+            except UnknownHandle:
+                out.append(None)
+        return out

halo_format-0.1.0/src/halo_format/summarize.py

@@ -0,0 +1,22 @@
+"""summarize(value, branches) -> str, deterministic.
+
+The summary is part of the hashed node, so it MUST be a pure function of structural facts and never
+LLM prose, or the same value would yield different handles and content-addressing breaks.
+derive_summary depends only on the branch names (count + sorted names), which makes it identical for
+encode, extend, and merge alike — none of those reconstruct the aggregate value.
+
+Names are sorted by UTF-16 code unit (via the UTF-16-BE encoding as the sort key), matching JCS key
+ordering, so the string is byte-identical to the TypeScript port (whose default sort is already by
+UTF-16 code unit). Plain code-point sorting would diverge on astral characters.
+"""
+
+
+def derive_summary(value, branches):
+    """Deterministic structural summary of a branch: "<n> branches: <sorted names>"."""
+    names = sorted(branches.keys(), key=lambda s: s.encode("utf-16-be"))
+    n = len(names)
+    if n == 0:
+        return "0 branches"
+    if n == 1:
+        return f"1 branch: {names[0]}"
+    return f"{n} branches: {', '.join(names)}"

halo_format-0.1.0/tests/test_canonical.py

@@ -0,0 +1,33 @@
+"""Unit tests for behavior the JSON conformance vectors cannot express.
+
+Non-finite numbers are not valid JSON, so they only matter at the encode API boundary; handle
+shape and key-order independence are properties of the primitives themselves. The shared
+vector-driven conformance lives in py/conformance.
+"""
+
+import math
+import re
+
+import pytest
+
+from halo_format import canonical, handle_of
+from halo_format.errors import CanonicalizationError
+
+
+@pytest.mark.parametrize("bad", [math.nan, math.inf, -math.inf])
+def test_rejects_non_finite(bad):
+    with pytest.raises(CanonicalizationError):
+        canonical(bad)
+
+
+def test_rejects_non_finite_nested():
+    with pytest.raises(CanonicalizationError):
+        canonical({"a": 1, "b": [2, math.inf]})
+
+
+def test_handle_shape():
+    assert re.fullmatch(r"h:[0-9a-f]{64}", handle_of({"hello": "world"}))
+
+
+def test_handle_is_key_order_independent():
+    assert handle_of({"a": 1, "b": 2}) == handle_of({"b": 2, "a": 1})

halo_format-0.1.0/tests/test_encode.py

@@ -0,0 +1,160 @@
+"""Unit tests for the encode pipeline: carve, summarize, encode, extend, merge. These defend the
+load-bearing invariants (determinism, content-addressing, Merkle integrity, dedup) that the shared
+whole-tree envelope vectors will later pin cross-port. CROSS_PORT_ROOT is the root handle the
+TypeScript port produces for the same input — asserting it here is a real cross-implementation lock
+without needing a vector file yet."""
+
+import pytest
+
+from halo_format import (
+    MemoryStore,
+    WrongKind,
+    auto_carve,
+    branch_node,
+    decode,
+    derive_summary,
+    encode,
+    extend,
+    is_branch,
+    leaf_node,
+    make_auto_carve,
+    merge,
+    node_handle,
+)
+
+# The credit-report-shaped fixture, with a `note` big enough to force the top object to carve.
+FIXTURE = {
+    "income": {"monthly": 4200},
+    "debts": {"monthly": 2604},
+    "inquiries": [1, 2, 3],
+    "tradelines": ["a", "b", "c", "d", "e", "f", "g"],
+    "note": "x" * 2000,
+}
+
+# Root handle the TypeScript port produces for FIXTURE — the cross-port interop lock.
+CROSS_PORT_ROOT = "h:41d7286f74ae2aeb6ca69823ddf25b9381dccba488602c1a8347a6b5eb163223"
+
+
+def test_auto_carve_inlines_small_and_primitive():
+    assert auto_carve(42, []) == {"as": "leaf"}
+    assert auto_carve("s", []) == {"as": "leaf"}
+    assert auto_carve(None, []) == {"as": "leaf"}
+    assert auto_carve({}, []) == {"as": "leaf"}
+    assert auto_carve([], []) == {"as": "leaf"}
+    assert auto_carve({"a": 1}, []) == {"as": "leaf"}  # small object inlines
+
+
+def test_auto_carve_carves_large_object():
+    big = {"a": "x" * 2000, "b": 1}
+    d = auto_carve(big, [])
+    assert d["as"] == "branch"
+    assert sorted(d["children"].keys()) == ["a", "b"]
+
+
+def test_auto_carve_chunks_long_array():
+    arr = list(range(60))
+    d = make_auto_carve(array_threshold=25)(arr, [])
+    assert d["as"] == "branch"
+    assert list(d["children"].keys()) == ["0", "1", "2"]
+    assert d["children"]["0"] == arr[0:25]
+    assert d["children"]["2"] == arr[50:60]
+
+
+def test_auto_carve_inline_bytes_zero_forces_branch():
+    assert make_auto_carve(inline_bytes=0)({"a": 1}, [])["as"] == "branch"
+
+
+def test_derive_summary_deterministic_and_sorted():
+    assert derive_summary(None, {}) == "0 branches"
+    assert derive_summary(None, {"child": "h:x"}) == "1 branch: child"
+    assert derive_summary(None, {"income": "h:1", "debts": "h:2"}) == "2 branches: debts, income"
+
+
+def test_encode_deterministic_and_cross_port():
+    a = encode(FIXTURE, MemoryStore())
+    b = encode(FIXTURE, MemoryStore())
+    assert a["handle"] == b["handle"]
+    assert a["handle"] == CROSS_PORT_ROOT
+    assert a["envelope"] == b["envelope"]
+
+
+def test_encode_envelope_shape():
+    store = MemoryStore()
+    r = encode(FIXTURE, store)
+    env = r["envelope"]
+    assert env["halo"] == "1"
+    assert env["alg"] == "sha256"
+    assert env["root"] == r["handle"]
+    assert env["view"]["summary"] == "5 branches: debts, income, inquiries, note, tradelines"
+    assert sorted(env["view"]["branches"].keys()) == [
+        "debts",
+        "income",
+        "inquiries",
+        "note",
+        "tradelines",
+    ]
+
+
+def test_encode_content_addressed_root():
+    store = MemoryStore()
+    r = encode(FIXTURE, store)
+    root_node = branch_node(r["envelope"]["view"]["summary"], r["envelope"]["view"]["branches"])
+    assert node_handle(root_node) == r["handle"]
+
+
+def test_encode_merkle_integrity():
+    changed = {**FIXTURE, "income": {"monthly": 4201}}
+    a = encode(FIXTURE, MemoryStore())
+    b = encode(changed, MemoryStore())
+    assert b["handle"] != a["handle"]
+
+
+def test_encode_dedup_equal_subtrees():
+    store = MemoryStore()
+    sub = {"blob": "y" * 2000}
+    encode({"left": sub, "right": sub, "pad": "z" * 2000}, store)
+    # root + 3 children, but left and right share one leaf -> 4 nodes, not 5
+    assert len(store) == 4
+
+
+def test_encode_leaf_root():
+    store = MemoryStore()
+    env = encode([1, 2, 3], store)["envelope"]
+    assert env["view"]["summary"] == "leaf: array of 3 items"
+    assert env["view"]["branches"] == {}
+
+
+def test_extend_reuses_children_and_retains_prior_root():
+    store = MemoryStore()
+    base = encode({"a": "x" * 2000, "b": "y" * 2000}, store)
+    size_before = len(store)
+    ext = extend(base["handle"], "c", "z" * 2000, store)
+    assert sorted(ext["envelope"]["view"]["branches"].keys()) == ["a", "b", "c"]
+    assert len(store) == size_before + 2  # one new leaf c + one new root
+    assert store.has(base["handle"])  # prior root survives
+
+
+def test_extend_last_write_wins():
+    store = MemoryStore()
+    base = encode({"a": "x" * 2000, "b": "y" * 2000}, store)
+    ext = extend(base["handle"], "a", "new", store)
+    a_handle = ext["envelope"]["view"]["branches"]["a"]
+    assert decode(store.get(a_handle)) == leaf_node("new")
+
+
+def test_extend_rejects_non_branch_root():
+    store = MemoryStore()
+    leaf = encode(42, store)  # primitive -> leaf root
+    with pytest.raises(WrongKind):
+        extend(leaf["handle"], "x", 1, store)
+
+
+def test_merge_combines_branch_tables_last_write_wins():
+    store = MemoryStore()
+    m1 = encode({"a": "x" * 2000, "b": "y" * 2000}, store)
+    m2 = encode({"b": "Y" * 2000, "c": "z" * 2000}, store)
+    merged = merge([m1["handle"], m2["handle"]], store)
+    assert sorted(merged["envelope"]["view"]["branches"].keys()) == ["a", "b", "c"]
+    m2_node = decode(store.get(m2["handle"]))
+    assert is_branch(m2_node)
+    assert merged["envelope"]["view"]["branches"]["b"] == m2_node["branches"]["b"]

halo_format-0.1.0/tests/test_navigate.py

@@ -0,0 +1,183 @@
+"""Unit tests for the navigator: verified walk/fetch/fetch_many, the trust chain (store-untrusted
+safety), ref resolution (raw handle, branch name, nested path, map-scoped via registered
+envelopes), and the encode -> open_ -> walk -> fetch round trip that is the L0+L1 milestone."""
+
+import pytest
+
+from halo_format import (
+    HashMismatch,
+    MemoryStore,
+    UnknownHandle,
+    WrongKind,
+    encode,
+    fetch,
+    fetch_many,
+    open_,
+    verify_envelope,
+    walk,
+)
+
+# Top object > 1024 bytes (via bio) so it carves; profile also carves; score/tags inline as leaves.
+DATA = {
+    "profile": {"name": "Ada", "city": "London", "bio": "x" * 2000},
+    "score": 612,
+    "tags": ["a", "b"],
+}
+
+
+def _encoded(value=DATA):
+    store = MemoryStore()
+    r = encode(value, store)
+    return store, r["handle"], r["envelope"]
+
+
+class TamperStore:
+    """Serves correct bytes for everything except one target handle, for which it returns
+    attacker-chosen bytes — exercises the store-untrusted guarantee. No get_many, so reads fall
+    back to get() and hit this override."""
+
+    def __init__(self, base, target, evil):
+        self._base = base
+        self._target = target
+        self._evil = evil
+
+    def put(self, data):
+        return self._base.put(data)
+
+    def get(self, handle):
+        return self._evil if handle == self._target else self._base.get(handle)
+
+    def has(self, handle):
+        return self._base.has(handle)
+
+
+def test_open_verifies_honest_branch_root():
+    store, _, env = _encoded()
+    nav = open_(env, store)
+    assert nav.root == env["root"]
+
+
+def test_open_leaf_root():
+    store, _, env = _encoded([1, 2, 3])
+    nav = open_(env, store)
+    assert nav.root == env["root"]
+
+
+def test_open_rejects_tampered_view():
+    store, _, env = _encoded()
+    tampered = {**env, "view": {**env["view"], "summary": "lies"}}
+    with pytest.raises(HashMismatch):
+        open_(tampered, store)
+
+
+def test_verify_envelope_true_and_source_stable():
+    _, _, env = _encoded()
+    assert verify_envelope(env) is True
+    a = {**env, "source": {"id": "m1"}}
+    b = {**env, "source": {"id": "m2", "tool": "t"}}
+    assert verify_envelope(a) is True
+    assert verify_envelope(b) is True
+
+
+def test_verify_envelope_false_for_tampered_view_and_leaf_root():
+    _, _, env = _encoded()
+    tampered = {**env, "view": {**env["view"], "summary": "lies"}}
+    assert verify_envelope(tampered) is False
+    _, _, leaf_env = _encoded(42)
+    assert verify_envelope(leaf_env) is False
+
+
+def test_free_walk_and_fetch_by_handle():
+    store, _, env = _encoded()
+    profile_handle = env["view"]["branches"]["profile"]
+    w = walk(profile_handle, store)
+    assert w["summary"] == "3 branches: bio, city, name"
+    assert fetch(w["branches"]["name"], store) == "Ada"
+
+
+def test_walk_on_leaf_and_fetch_on_branch_raise_wrongkind():
+    store, _, env = _encoded()
+    with pytest.raises(WrongKind):
+        walk(env["view"]["branches"]["score"], store)  # leaf
+    with pytest.raises(WrongKind):
+        fetch(env["view"]["branches"]["profile"], store)  # branch
+
+
+def test_unknown_handle():
+    store, _, _ = _encoded()
+    with pytest.raises(UnknownHandle):
+        fetch("h:" + "0" * 64, store)
+
+
+def test_store_untrusted_substituted_leaf_fails():
+    store, _, env = _encoded()
+    score_handle = env["view"]["branches"]["score"]
+    evil = b'{"k":"l","value":999}'
+    evil_store = TamperStore(store, score_handle, evil)
+    with pytest.raises(HashMismatch):
+        fetch(score_handle, evil_store)
+
+
+def test_navigator_resolves_name_nested_and_raw_handle():
+    store, _, env = _encoded()
+    nav = open_(env, store)
+    assert nav.fetch("score") == 612
+    assert nav.fetch("profile.name") == "Ada"
+    assert nav.fetch("profile.city") == "London"
+    w = nav.walk("profile")
+    assert sorted(w["branches"].keys()) == ["bio", "city", "name"]
+    assert nav.fetch(env["view"]["branches"]["score"]) == 612  # raw handle
+
+
+def test_navigator_rejects_leaf_descent_and_unknown_name():
+    store, _, env = _encoded()
+    nav = open_(env, store)
+    with pytest.raises(WrongKind):
+        nav.fetch("score.nope")
+    with pytest.raises(UnknownHandle):
+        nav.fetch("missing")
+
+
+def test_navigator_map_scoped_refs():
+    store = MemoryStore()
+    a = encode({"score": 1, "pad": "x" * 2000}, store)
+    b = encode({"score": 2, "pad": "y" * 2000}, store)
+    env1 = {**a["envelope"], "source": {"id": "m1"}}
+    env2 = {**b["envelope"], "source": {"id": "m2"}}
+    nav = open_(env1, store).register(env2)
+    assert nav.fetch("m1.score") == 1
+    assert nav.fetch("m2.score") == 2
+    assert nav.fetch("score") == 1  # bare name -> primary (m1)
+
+
+def test_fetch_many_batches_with_per_ref_results():
+    store, _, env = _encoded()
+    score_h = env["view"]["branches"]["score"]
+    profile_h = env["view"]["branches"]["profile"]
+    missing = "h:" + "0" * 64
+    out = fetch_many([score_h, profile_h, missing], store)
+    assert out[score_h] == {"ok": True, "value": 612}
+    assert out[profile_h] == {"ok": False, "error": "WrongKind"}
+    assert out[missing] == {"ok": False, "error": "UnknownHandle"}
+
+
+def test_navigator_fetch_many_surfaces_hashmismatch_per_entry():
+    store, _, env = _encoded()
+    score_handle = env["view"]["branches"]["score"]
+    evil = b'{"k":"l","value":999}'
+    evil_nav = open_(env, TamperStore(store, score_handle, evil))
+    out = evil_nav.fetch_many(["score", "profile.name", "missing"])
+    assert out["score"] == {"ok": False, "error": "HashMismatch"}
+    assert out["profile.name"] == {"ok": True, "value": "Ada"}
+    assert out["missing"] == {"ok": False, "error": "UnknownHandle"}
+
+
+def test_round_trip_milestone():
+    store, _, env = _encoded()
+    nav = open_(env, store)
+    out = nav.fetch_many(["profile.name", "profile.city", "score"])
+    assert out == {
+        "profile.name": {"ok": True, "value": "Ada"},
+        "profile.city": {"ok": True, "value": "London"},
+        "score": {"ok": True, "value": 612},
+    }

halo_format-0.1.0/tests/test_node.py

@@ -0,0 +1,51 @@
+"""Unit tests for node decode validation and the leaf/branch kind-tag separation. Happy-path
+serialize/handle/round-trip behavior is covered by the shared node vectors in py/conformance."""
+
+import pytest
+
+from halo_format import (
+    HaloError,
+    branch_node,
+    decode,
+    is_branch,
+    is_leaf,
+    leaf_node,
+    node_handle,
+)
+
+
+def test_leaf_constructor_and_guards():
+    n = leaf_node(42)
+    assert n == {"k": "l", "value": 42}
+    assert is_leaf(n) is True
+    assert is_branch(n) is False
+
+
+def test_branch_constructor_defensive_copy():
+    branches = {"a": "h:aa"}
+    n = branch_node("one", branches)
+    branches["a"] = "h:bb"  # mutate caller's map
+    assert n["branches"]["a"] == "h:aa"  # node unaffected
+    assert is_branch(n) is True
+
+
+@pytest.mark.parametrize(
+    "raw",
+    [
+        b"{bad",                                       # invalid JSON
+        b"[1,2]",                                      # not an object
+        b'{"k":"x"}',                                  # unknown kind
+        b'{"k":"l"}',                                  # leaf missing value
+        b'{"k":"b","summary":1,"branches":{}}',        # non-string summary
+        b'{"k":"b","summary":"s","branches":{"a":1}}',  # non-string child handle
+    ],
+)
+def test_decode_rejects_malformed(raw):
+    with pytest.raises(HaloError):
+        decode(raw)
+
+
+def test_kind_tag_prevents_collision():
+    leaf = leaf_node({"summary": "x", "branches": {}})
+    branch = branch_node("x", {})
+    assert node_handle(leaf) != node_handle(branch)

halo_format-0.1.0/tests/test_store.py

@@ -0,0 +1,61 @@
+"""Unit tests for the stores: content-addressing, idempotent dedup, immutability, missing
+handles, batch reads, filesystem persistence, and cross-store handle agreement."""
+
+import re
+
+from halo_format import FileStore, MemoryStore, UnknownHandle
+
+MISSING = "h:" + "0" * 64
+
+
+def test_memory_put_dedups_identical_bytes():
+    s = MemoryStore()
+    h1 = s.put(b"hello")
+    h2 = s.put(b"hello")
+    assert h1 == h2
+    assert re.fullmatch(r"h:[0-9a-f]{64}", h1)
+    assert len(s) == 1  # stored once
+
+
+def test_memory_get_and_unknown_handle():
+    s = MemoryStore()
+    h = s.put(b"x")
+    assert s.get(h) == b"x"
+    try:
+        s.get(MISSING)
+        assert False, "expected UnknownHandle"
+    except UnknownHandle:
+        pass
+
+
+def test_memory_has():
+    s = MemoryStore()
+    h = s.put(b"y")
+    assert s.has(h) is True
+    assert s.has(MISSING) is False
+
+
+def test_memory_get_many_missing_in_place():
+    s = MemoryStore()
+    h = s.put(b"present")
+    assert s.get_many([h, MISSING]) == [b"present", None]
+
+
+def test_filestore_roundtrip_and_persistence(tmp_path):
+    a = FileStore(str(tmp_path))
+    h = a.put(b"persisted")
+
+    b = FileStore(str(tmp_path))  # fresh instance, same dir
+    assert b.has(h) is True
+    assert b.get(h) == b"persisted"
+    try:
+        b.get(MISSING)
+        assert False, "expected UnknownHandle"
+    except UnknownHandle:
+        pass
+    assert b.get_many([h, MISSING]) == [b"persisted", None]
+
+
+def test_stores_agree_on_handle(tmp_path):
+    data = b"content-addressed regardless of store"
+    assert MemoryStore().put(data) == FileStore(str(tmp_path)).put(data)