yaffle 0.0.1__py3-none-any.whl

yaffle/__init__.py

@@ -0,0 +1,77 @@
+"""yaffle: the runtime of the Python code the yaffle compiler generates.
+
+Generated modules import it as ``_rt``. Pure Python 3.12+, no dependencies. Besides what
+generated code calls, it offers sources (:func:`file_source`), errors (:class:`YaffleError`),
+checksums and canonical JSON helpers.
+
+Modules: ``errors``, ``prim`` (primitive encodings), ``ints`` (number semantics), ``strings``,
+``checksum``, ``arrays`` (array builtins and equality), ``jsonc`` (canonical JSON), ``values``
+(checks of values), ``externs`` (host code), ``codec`` (``@via`` and codec reuse), ``streams``
+(streams), ``reader``, ``writer`` and ``layout`` (parse and serialize), ``api`` (root API
+glue and sources), and for views ``store`` (bytes read on demand), ``view``, ``commit`` and
+``async_view``.
+"""
+
+from __future__ import annotations
+
+from . import (
+    api,
+    arrays,
+    async_view,
+    checksum,
+    codec,
+    commit,
+    errors,
+    externs,
+    ints,
+    jsonc,
+    layout,
+    prim,
+    reader,
+    streams,
+    strings,
+    values,
+    view,
+    writer,
+)
+from .api import *
+from .arrays import *
+from .async_view import *
+from .checksum import *
+from .codec import *
+from .commit import *
+from .errors import *
+from .externs import *
+from .ints import *
+from .jsonc import *
+from .layout import *
+from .prim import *
+from .reader import *
+from .streams import *
+from .strings import *
+from .values import *
+from .view import *
+from .writer import *
+
+__version__ = "0.0.1"
+
+__all__ = [
+    *api.__all__,
+    *arrays.__all__,
+    *async_view.__all__,
+    *checksum.__all__,
+    *codec.__all__,
+    *commit.__all__,
+    *errors.__all__,
+    *externs.__all__,
+    *ints.__all__,
+    *jsonc.__all__,
+    *layout.__all__,
+    *prim.__all__,
+    *reader.__all__,
+    *streams.__all__,
+    *strings.__all__,
+    *values.__all__,
+    *view.__all__,
+    *writer.__all__,
+]

yaffle/_harness.py

@@ -0,0 +1,288 @@
+"""Conformance harness: runs a batch of conformance cases through generated code.
+
+Usage: ``python -m yaffle._harness <spec.json> <results-dir>``
+
+The spec (written by the Python backend's conformance adapter) lists the cases of one case
+directory: which generated module and class to use, the input bytes (hex or a file), the JSON,
+and what is expected. One result file per case is written to the results directory
+(``<index>.json``), with the actual canonical JSON, output bytes and error, and the harness's own
+verdict (the runner re-checks them). See docs/CONFORMANCE.md.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import importlib
+import json
+import math
+import os
+import re
+import sys
+import traceback
+from typing import Any
+
+from .api import file_source
+from .commit import apply_patches
+from .errors import YaffleError
+
+_NOT_IMPLEMENTED = re.compile(r"not implemented|not supported", re.IGNORECASE)
+
+
+class _Failed(Exception):
+    """Ends a case with a message (not an error the case could expect)."""
+
+
+def _type_name(v: Any) -> str:
+    if v is None:
+        return "null"
+    if isinstance(v, bool):
+        return "boolean"
+    if isinstance(v, (int, float)):
+        return "number"
+    if isinstance(v, str):
+        return "string"
+    if isinstance(v, list):
+        return "array"
+    return "object"
+
+
+def _show(v: Any) -> str:
+    s = json.dumps(v)
+    return s if len(s) <= 120 else s[:120] + "…"
+
+
+def json_diff(actual: Any, expected: Any, path: str = "") -> str | None:
+    """The first difference between two canonical JSON values (key order significant), or None."""
+    where = path or "/"
+    ta, te = _type_name(actual), _type_name(expected)
+    if ta != te:
+        return f"{where}: expected {_show(expected)}, got {_show(actual)}"
+    if ta == "array":
+        if len(actual) != len(expected):
+            return f"{where}: expected {len(expected)} elements, got {len(actual)}"
+        for i, (a, e) in enumerate(zip(actual, expected)):
+            d = json_diff(a, e, f"{path}/{i}")
+            if d:
+                return d
+        return None
+    if ta == "object":
+        for k in expected:
+            if k not in actual:
+                return f"{path}/{k}: missing"
+        for k in actual:
+            if k not in expected:
+                return f"{path}/{k}: unexpected"
+        for k in expected:
+            d = json_diff(actual[k], expected[k], f"{path}/{k}")
+            if d:
+                return d
+        if list(actual) != list(expected):
+            return f"{where}: fields are not in declaration order"
+        return None
+    if ta == "number":
+        a, e = float(actual), float(expected)
+        if a != e or (a == 0 and math.copysign(1.0, a) != math.copysign(1.0, e)):
+            return f"{where}: expected {_show(expected)}, got {_show(actual)}"
+        return None
+    if actual != expected:
+        return f"{where}: expected {_show(expected)}, got {_show(actual)}"
+    return None
+
+
+def _load_bytes(spec: Any) -> bytes | None:
+    if spec is None:
+        return None
+    if isinstance(spec, str):
+        return bytes.fromhex(re.sub(r"[\s_]+", "", spec))
+    with open(spec["file"], "rb") as f:
+        return f.read()
+
+
+def _options(cls: Any, c: dict[str, Any]) -> dict[str, Any]:
+    options: dict[str, Any] = cls.options_from_json(c.get("args") or {})
+    if c.get("strict"):
+        options["strict"] = True
+    return options
+
+
+class _Case:
+    """One case: its spec, its generated class and the result being built."""
+
+    def __init__(self, c: dict[str, Any], cls: Any, max_hex: int) -> None:
+        self.c = c
+        self.cls = cls
+        self.max_hex = max_hex
+        self.is_async = bool(c.get("async"))
+        self.error: Any = c.get("error")
+        self.result: dict[str, Any] = {"name": c["name"], "ok": False}
+        self.phase = "serialize" if c.get("input") is not None else "parse"
+
+    def set_bytes(self, out: bytes) -> None:
+        if len(out) <= self.max_hex:
+            self.result["bytes"] = bytes(out).hex()
+
+    def expect_no_error(self, phase: str | None, what: str) -> None:
+        """Fails the case if it expects an error (in ``phase``, if given): ``what`` succeeded."""
+        if self.error and (phase is None or self.error.get("phase") == phase):
+            raise _Failed(f"expected {self.error['code']}, but {what} succeeded")
+
+    def check_json(self, actual: Any, mode: str, what: str) -> None:
+        """Records the JSON read; fails if a parse error was expected or the JSON differs."""
+        self.result["json"] = actual
+        self.expect_no_error("parse", what)
+        expected = self.c.get("json")
+        if expected is not None:
+            d = json_diff(actual, expected)
+            if d:
+                raise _Failed(f"{mode}: {d}")
+
+    async def parse(self, data: bytes, options: dict[str, Any]) -> Any:
+        if self.is_async:
+            return await self.cls.parse_async(data, **options)
+        return self.cls.parse(data, **options)
+
+    async def serialize(self, value: Any, options: dict[str, Any]) -> bytes:
+        write_options = {k: v for k, v in options.items() if k != "strict"}
+        if self.is_async:
+            out: bytes = await self.cls.serialize_async(value, **write_options)
+        else:
+            out = self.cls.serialize(value, **write_options)
+        self.set_bytes(out)
+        return out
+
+    async def run(self) -> None:
+        c = self.c
+        options = _options(self.cls, c)
+        data = _load_bytes(c.get("bytes"))
+        if c.get("input") is not None:
+            out = await self.serialize(self.cls.from_json(c["input"]), options)
+            self.expect_no_error(None, "serialize")
+            if data is None:
+                raise _Failed("serialize case without bytes")
+            if out != data:
+                raise _Failed(f"serialize: expected {data.hex()}, got {self.result.get('bytes', '(large)')}")
+            return
+        if data is None:
+            raise _Failed("parse case without bytes")
+        value = await self.parse(data, options)
+        self.check_json(value.to_json(), "parse", "parse")
+        # Cases expecting a serialize error round-trip the JSON.
+        rt = "json" if (self.error and self.error.get("phase") == "serialize") else c.get("roundtrip", True)
+        if rt is False:
+            return
+        self.phase = "serialize"
+        source = value if rt == "value" else self.cls.from_json(c["json"] if c.get("json") is not None else self.result["json"])
+        out = await self.serialize(source, options)
+        self.expect_no_error("serialize", "serialize")
+        if out != data:
+            i = 0
+            n = min(len(out), len(data))
+            while i < n and out[i] == data[i]:
+                i += 1
+            raise _Failed(f"roundtrip: bytes differ at offset 0x{i:x} (expected {len(data)} bytes, got {len(out)})")
+
+    async def run_view(self) -> None:
+        """View mode (docs/CONFORMANCE.md §7): read every field through the lazy view, then commit
+        with no edits (which must reproduce the input)."""
+        c = self.c
+        options = _options(self.cls, c)
+        spec = c.get("bytes")
+        if spec is None:
+            raise _Failed("view case without bytes")
+        # A file is viewed through a memory-mapped source.
+        src: Any = _load_bytes(spec) if isinstance(spec, str) else file_source(spec["file"])
+        try:
+            if self.is_async:
+                v = self.cls.view_async(src, **options)
+                actual = await v._to_json()
+            else:
+                v = self.cls.view(src, **options)
+                actual = self.cls.to_json(v)  # reads every field through the view
+            self.check_json(actual, "view", "reading the view")
+            if c.get("roundtrip", True) is False:
+                return
+            self.phase = "serialize"
+            orig = _load_bytes(spec)
+            assert orig is not None
+            if self.is_async:
+                out = apply_patches(orig, await v._commit())
+            else:
+                out = v._commit()
+            self.set_bytes(out)
+            if orig != bytes(out):
+                raise _Failed(f"commit with no edits changed the bytes ({len(orig)} → {len(out)} bytes)")
+        finally:
+            close = getattr(src, "close", None)
+            if callable(close):
+                close()
+
+    def failed(self, e: BaseException) -> None:
+        """Records that the case raised ``e`` in the current phase."""
+        result = self.result
+        if not isinstance(e, YaffleError):
+            msg = str(e)
+            if _NOT_IMPLEMENTED.search(msg):
+                result["skipped"] = msg
+            else:
+                result["message"] = f"unexpected exception: {type(e).__name__}: {msg}\n{traceback.format_exc()}"
+            return
+        message = e.describe()
+        err = {"phase": self.phase, "code": e.code, "path": list(e.path), "offset": e.offset, "message": message}
+        result["error"] = err
+        expected = self.error
+        if not expected:
+            if _NOT_IMPLEMENTED.search(message):
+                result["skipped"] = message
+            else:
+                result["message"] = f"unexpected {e.code} error in {self.phase}: {message}"
+        elif expected.get("phase") != self.phase:
+            result["message"] = f"expected a {expected['phase']} error, got {e.code} in {self.phase}: {message}"
+        elif e.code != expected.get("code"):
+            result["message"] = f"expected {expected['code']}, got {e.code}: {message}"
+        elif expected.get("path") is not None and [str(x) for x in e.path] != [str(x) for x in expected["path"]]:
+            result["message"] = f"expected path {expected['path']}, got {list(e.path)} ({message})"
+        else:
+            result["ok"] = True
+
+
+async def run_case(c: dict[str, Any], max_hex: int, mode: str = "parse") -> dict[str, Any]:
+    try:
+        cls = getattr(importlib.import_module(c["module"]), c["cls"])
+    except Exception as e:  # pragma: no cover - generated code that doesn't import
+        return {"name": c["name"], "ok": False, "message": f"cannot import generated code: {e}\n{traceback.format_exc()}"}
+    case = _Case(c, cls, max_hex)
+    try:
+        await (case.run_view() if mode == "view" else case.run())
+        case.result["ok"] = True
+    except _Failed as f:
+        case.result["message"] = str(f)
+    except (KeyboardInterrupt, SystemExit):
+        raise
+    except BaseException as e:  # noqa: BLE001 - every failure is reported
+        case.failed(e)
+    return case.result
+
+
+async def main(spec_path: str, out_dir: str) -> None:
+    with open(spec_path, encoding="utf-8") as f:
+        spec = json.load(f)
+    # Real files nest deeply.
+    sys.setrecursionlimit(max(sys.getrecursionlimit(), 200_000))
+    os.makedirs(out_dir, exist_ok=True)
+    max_hex = int(spec.get("maxHexBytes", 1 << 20))
+    mode = str(spec.get("mode", "parse"))
+    for i, c in enumerate(spec["cases"]):
+        res = await run_case(c, max_hex, mode)
+        tmp = os.path.join(out_dir, f"{i}.json.tmp")
+        with open(tmp, "w", encoding="utf-8") as f:
+            try:
+                json.dump(res, f, allow_nan=False)
+            except (TypeError, ValueError) as e:
+                f.seek(0)
+                f.truncate()
+                json.dump({"name": c["name"], "ok": False, "message": f"result is not JSON: {e}"}, f)
+        os.replace(tmp, os.path.join(out_dir, f"{i}.json"))
+
+
+if __name__ == "__main__":
+    asyncio.run(main(sys.argv[1], sys.argv[2]))

yaffle/api.py

@@ -0,0 +1,212 @@
+"""What the generated root APIs (``X.parse``, ``X.serialize``, …) run on, and byte sources.
+
+A source is a bytes-like object, or an object with a ``size`` and ``read(offset, length)``. If
+``read`` returns an awaitable, the source is async: use ``parse_async`` (or an async view).
+"""
+
+from __future__ import annotations
+
+import inspect
+import mmap
+import os
+import sys
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable, Generic, Literal, Protocol, Sequence, TypeVar, Union
+
+from .errors import YaffleError, fail
+from .reader import Reader
+from .writer import Region, Writer
+
+__all__ = [
+    "AsyncSource", "FileSource", "SafeResult", "Source", "SyncSource", "endian_option",
+    "file_source", "parse_with", "parse_with_async", "read_all", "read_all_async", "safe",
+    "safe_async", "serialize_with", "serialize_with_async",
+]  # fmt: skip
+
+T = TypeVar("T")
+
+
+class SyncSource(Protocol):
+    @property
+    def size(self) -> int: ...
+
+    def read(self, offset: int, length: int) -> bytes: ...
+
+
+class AsyncSource(Protocol):
+    @property
+    def size(self) -> int: ...
+
+    def read(self, offset: int, length: int) -> Awaitable[bytes]: ...
+
+
+Source = Union[bytes, bytearray, memoryview, SyncSource, AsyncSource]
+
+
+@dataclass
+class SafeResult(Generic[T]):
+    """The result of ``safe_parse``: ``ok`` with a ``value``, or not ``ok`` with an ``error``."""
+
+    ok: bool
+    value: T | None = None
+    error: YaffleError | None = None
+
+
+def ensure_recursion(limit: int = 100_000) -> None:
+    """Raises the interpreter's recursion limit for deeply nested data (never lowers it)."""
+    if sys.getrecursionlimit() < limit:
+        sys.setrecursionlimit(limit)
+
+
+def reject_path(source: Any) -> None:
+    if isinstance(source, (str, os.PathLike)):
+        fail("INPUT", "pass bytes or a source, not a path (use file_source(path))")
+
+
+def read_all(source: Any) -> bytes:
+    """Reads a whole sync source into memory."""
+    if isinstance(source, (bytes, bytearray, memoryview)):
+        return bytes(source)
+    reject_path(source)
+    out = source.read(0, source.size)
+    if inspect.isawaitable(out):
+        if inspect.iscoroutine(out):
+            out.close()
+        fail("INPUT", "this source is async; use parse_async or an async view")
+    return bytes(out)
+
+
+async def read_all_async(source: Any) -> bytes:
+    """Reads a whole source (sync or async) into memory."""
+    if isinstance(source, (bytes, bytearray, memoryview)):
+        return bytes(source)
+    out = source.read(0, source.size)
+    if inspect.isawaitable(out):
+        out = await out
+    return bytes(out)
+
+
+def parse_with(source: Any, strict: bool, read: Callable[[Reader], T]) -> T:
+    """Runs a generated root reader over a sync source."""
+    ensure_recursion()
+    return read(Reader(read_all(source), strict))
+
+
+async def parse_with_async(source: Any, strict: bool, read: Callable[[Reader], Any]) -> Any:
+    """Runs a generated root reader (sync or async) over any source."""
+    ensure_recursion()
+    out = read(Reader(await read_all_async(source), strict))
+    return await out if inspect.isawaitable(out) else out
+
+
+def serialize_with(layout: Sequence[Any] | None, write: Callable[[Writer], Any]) -> bytes:
+    """Runs a generated root writer in a fresh root region and returns the bytes."""
+    ensure_recursion()
+    region = Region(layout)
+    region.attach_root(write(Writer(region)))
+    return bytes(region.finish())
+
+
+async def serialize_with_async(
+    layout: Sequence[Any] | None, write: Callable[[Writer], Awaitable[Any]]
+) -> bytes:
+    ensure_recursion()
+    region = Region(layout)
+    region.attach_root(await write(Writer(region)))
+    return bytes(region.finish())
+
+
+def safe(fn: Callable[[], T]) -> SafeResult[T]:
+    """Runs ``fn``; a raised :class:`YaffleError` becomes ``SafeResult(ok=False, error=…)``."""
+    try:
+        return SafeResult(True, fn())
+    except YaffleError as e:
+        return SafeResult(False, None, e)
+
+
+async def safe_async(fn: Callable[[], Awaitable[T]]) -> SafeResult[T]:
+    try:
+        return SafeResult(True, await fn())
+    except YaffleError as e:
+        return SafeResult(False, None, e)
+
+
+def endian_option(v: Any) -> Literal["le", "be"]:
+    if v == "le":
+        return "le"
+    if v == "be":
+        return "be"
+    fail("INPUT", f'endian must be "le" or "be", got {v!r}')
+
+
+class FileSource:
+    """A sync random-access source over a file (``size``, ``read(offset, length)``).
+
+    ``buffer()`` memory-maps the file, so views read only the pages they touch. With
+    ``writable=True`` the source also has ``write`` and ``truncate``, and a view's ``_commit``
+    writes the edits back to the file.
+    """
+
+    def __init__(self, path: str | os.PathLike[str], writable: bool = False) -> None:
+        self._f = open(path, "r+b" if writable else "rb")  # noqa: SIM115
+        self._f.seek(0, os.SEEK_END)
+        self.size = self._f.tell()
+        self.writable = writable
+        self._map: mmap.mmap | None = None
+        if not writable:
+            # Views write commits back to sources that have `write`.
+            self.write = None  # type: ignore[assignment, method-assign]
+
+    def read(self, offset: int, length: int) -> bytes:
+        self._f.seek(offset)
+        data = self._f.read(length)
+        if len(data) != length:
+            fail("EOF", "short read from file", offset)
+        return data
+
+    def buffer(self) -> Any:
+        """The file's bytes as a read-only memory map (bytes for an empty file)."""
+        if self.size == 0:
+            return b""
+        if self._map is None:
+            self._map = mmap.mmap(self._f.fileno(), 0, access=mmap.ACCESS_READ)
+        return self._map
+
+    def copy_buffer(self) -> Any:
+        """A private copy-on-write map of the file: a view's in-place patches copy only the pages
+        they write, and the file is not changed."""
+        if self.size == 0:
+            return bytearray()
+        return mmap.mmap(self._f.fileno(), 0, access=mmap.ACCESS_COPY)
+
+    def write(self, offset: int, data: bytes) -> None:
+        self._drop_map()
+        self._f.seek(offset)
+        self._f.write(data)
+        self._f.flush()
+        self.size = max(self.size, offset + len(data))
+
+    def truncate(self, n: int) -> None:
+        self._drop_map()
+        self._f.truncate(n)
+        self.size = n
+
+    def _drop_map(self) -> None:
+        if self._map is not None:
+            self._map.close()
+            self._map = None
+
+    def close(self) -> None:
+        self._drop_map()
+        self._f.close()
+
+    def __enter__(self) -> FileSource:
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()
+
+
+def file_source(path: str | os.PathLike[str], writable: bool = False) -> FileSource:
+    """A source over a file (memory-mapped for views; ``writable`` for committing views back)."""
+    return FileSource(path, writable)