cairns 0.2.0__py3-none-any.whl

cairns/__init__.py

@@ -0,0 +1,50 @@
+"""Cairns: compute graph orchestration with caching and observability."""
+
+from cairns.core import (
+    Cairn,
+    Handle,
+    Record,
+    Run,
+    Runtime,
+    cached_output,
+    cached_tracing,
+    cairn,
+    default_runtime,
+    step,
+    trace,
+)
+from cairns.patterns import rate_limited, replayable
+from cairns.run import (
+    arun,
+    gc,
+    list_runs,
+    remove_run,
+    remove_runs_before,
+    run,
+)
+
+__all__ = [
+    # canonical
+    "step",
+    "trace",
+    "Handle",
+    "run",
+    "arun",
+    "Runtime",
+    "default_runtime",
+    "Cairn",
+    "cairn",
+    "Record",
+    # tier-3 primitive (advanced)
+    "Run",
+    # batteries
+    "cached_output",
+    "cached_tracing",
+    "rate_limited",
+    "replayable",
+    # ops
+    "gc",
+    "list_runs",
+    "remove_run",
+    "remove_runs_before",
+]

cairns/cli/__init__.py

@@ -0,0 +1,219 @@
+"""Cairns CLI.
+
+Usage:
+    cairns script.py [ENTRY]            Run a script (default action)
+    cairns                              Browse runs interactively
+    cairns list                         List all runs
+    cairns show [RUN_ID]                Show trace (latest if omitted)
+    cairns output PATH                  Show a cached output
+    cairns gc [--before DATE]           Garbage collect
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import os
+import sys
+from datetime import datetime, timezone
+from typing import Any
+
+
+def _store_path(args: argparse.Namespace) -> str:
+    return getattr(args, "store", None) or ".cairns"
+
+
+# ── Commands ──
+
+
+def cmd_list(args: argparse.Namespace) -> None:
+    from cairns.run import show_runs
+
+    show_runs(_store_path(args))
+
+
+def cmd_show(args: argparse.Namespace) -> None:
+    from cairns.run import show_trace
+
+    run_id: str | None = getattr(args, "run_id", None)
+    show_trace(_store_path(args), run_id=run_id)
+
+
+def cmd_output(args: argparse.Namespace) -> None:
+    from cairns.run import show_output
+
+    path: str = args.path
+    if os.path.islink(path):
+        path = str(os.path.realpath(path))
+    show_output(path)
+
+
+def cmd_gc(args: argparse.Namespace) -> None:
+    from cairns.run import gc, list_runs
+
+    store = _store_path(args)
+    before: datetime | None = None
+    if args.before:
+        before = datetime.fromisoformat(args.before).replace(tzinfo=timezone.utc)
+
+    keep_latest: bool = args.keep_latest
+
+    # Show current state first
+    runs = list_runs(store)
+    if runs:
+        from cairns.run import show_runs
+        show_runs(store)
+
+    removed_runs, removed_outputs = gc(store, before=before, keep_latest=keep_latest)
+
+    if removed_runs:
+        print(f"Removed {len(removed_runs)} run(s):")
+        for r in removed_runs:
+            print(f"  {r}")
+    if removed_outputs:
+        print(f"Removed {len(removed_outputs)} orphaned output(s)")
+    if not removed_runs and not removed_outputs:
+        print("Nothing to clean up.")
+
+
+def cmd_run(script: str, entry_name: str, store: str, *, force: bool = False) -> None:
+    from cairns.run import run as cairn_run
+
+    # Load the script as a module
+    script_dir = os.path.dirname(os.path.abspath(script))
+    if script_dir not in sys.path:
+        sys.path.insert(0, script_dir)
+
+    spec = importlib.util.spec_from_file_location("__cairn_script__", script)
+    if spec is None or spec.loader is None:
+        print(f"Error: cannot load {script}", file=sys.stderr)
+        sys.exit(1)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["__cairn_script__"] = module
+    spec.loader.exec_module(module)
+
+    # Find the entry point: try explicit name, then 'main', then script basename
+    entry: Any = getattr(module, entry_name, None)
+    if entry is None and entry_name == "main":
+        basename = os.path.splitext(os.path.basename(script))[0]
+        entry = getattr(module, basename, None)
+        if entry is not None:
+            entry_name = basename
+    if entry is None:
+        candidates = [
+            name for name in dir(module)
+            if not name.startswith("_") and callable(getattr(module, name))
+        ]
+        print(f"Error: {script} has no function '{entry_name}'", file=sys.stderr)
+        if candidates:
+            print(f"Available functions: {', '.join(candidates)}", file=sys.stderr)
+        sys.exit(1)
+
+    # Build label from script path + entry name
+    script_rel = os.path.relpath(script)
+    script_module = os.path.splitext(script_rel)[0].replace(os.sep, ".")
+    label = f"{script_module}:{entry_name}"
+
+    # --force: remove previous runs for this entry point + GC orphaned outputs
+    if force:
+        from cairns.run import gc_outputs, list_runs, remove_run
+        runs = [r for r in list_runs(store) if r.entry_name == label]
+        for r in runs:
+            remove_run(store, r.run_id)
+        removed = gc_outputs(store)
+        if runs or removed:
+            print(f"Force: removed {len(runs)} run(s), {len(removed)} output(s)", file=sys.stderr)
+
+    try:
+        from cairns.tui import run_app
+        run_app(entry, store_path=store, label=label)
+    except ImportError:
+        # Fallback to headless mode
+        print(f"Running {script}:{entry_name}", file=sys.stderr)
+        print(f"Store: {store}/\n", file=sys.stderr)
+        try:
+            result = cairn_run(entry, store_path=store, label=label)
+            print(f"\nResult: {result}", file=sys.stderr)
+        except Exception as e:
+            print(f"\nError: {e}", file=sys.stderr)
+            sys.exit(1)
+
+
+def cmd_browse(store: str) -> None:
+    """Interactive run browser using Textual TUI."""
+    try:
+        from cairns.tui import browse
+        browse(store)
+    except ImportError:
+        # Fallback: just list runs
+        from cairns.run import show_runs
+        show_runs(store)
+        print("Install cairn[tui] for interactive browsing: uv pip install cairn[tui]")
+
+
+# ── Main ──
+
+
+def main() -> None:
+    # Quick check: is the first arg a .py file? → run it directly
+    if len(sys.argv) >= 2 and not sys.argv[1].startswith("-"):
+        first_arg = sys.argv[1]
+        if first_arg.endswith(".py") or os.path.isfile(first_arg):
+            # cairns script.py [entry] [--store PATH] [--force]
+            parser = argparse.ArgumentParser(prog="cairns")
+            parser.add_argument("script", help="Python script to run")
+            parser.add_argument("entry", nargs="?", default="main", help="Entry point function")
+            parser.add_argument("--store", "-s", default=".cairns")
+            parser.add_argument("--force", "-f", action="store_true", help="Clear cache for this entry point before running")
+            args = parser.parse_args()
+            cmd_run(args.script, args.entry, args.store, force=args.force)
+            return
+
+    # Otherwise: subcommand mode
+    parser = argparse.ArgumentParser(
+        prog="cairns",
+        description="Compute graph orchestration with caching and observability",
+    )
+    parser.add_argument("--store", "-s", default=".cairns", help="Store path (default: .cairns)")
+    subparsers = parser.add_subparsers(dest="command")
+
+    # cairns list
+    subparsers.add_parser("list", help="List all runs")
+
+    # cairns show [RUN_ID]
+    p_show = subparsers.add_parser("show", help="Show trace (latest if no run_id)")
+    p_show.add_argument("run_id", nargs="?", default=None)
+
+    # cairns output PATH
+    p_output = subparsers.add_parser("output", help="Show a cached output")
+    p_output.add_argument("path")
+
+    # cairns gc
+    p_gc = subparsers.add_parser("gc", help="Garbage collect")
+    p_gc.add_argument("--before", help="Remove runs before this ISO date")
+    p_gc.add_argument("--keep-latest", action="store_true", default=True)
+    p_gc.add_argument("--no-keep-latest", dest="keep_latest", action="store_false")
+
+    args = parser.parse_args()
+
+    commands: dict[str, Any] = {
+        "list": cmd_list,
+        "show": cmd_show,
+        "output": cmd_output,
+        "gc": cmd_gc,
+    }
+
+    if args.command is None:
+        # cairns with no args → interactive browser
+        cmd_browse(args.store)
+        return
+
+    cmd = commands.get(args.command)
+    if cmd is None:
+        parser.print_help()
+        sys.exit(1)
+    cmd(args)
+
+
+if __name__ == "__main__":
+    main()

cairns/core/__init__.py

@@ -0,0 +1,91 @@
+"""Cairn core primitives.
+
+Public surface of `cairns.core` — re-exported from dedicated submodules so
+external code can say `from cairns.core import step` without knowing where
+each name lives.
+"""
+
+from .step import (
+    Handle,
+    cached_output,
+    cached_tracing,
+    step,
+    trace,
+)
+from .runtime import (
+    Runtime,
+    Event,
+    InteractionSink,
+    MemorySink,
+    NullSink,
+    Run,
+    Sink,
+    current_run,
+    current_span,
+    default_runtime,
+    emit_event,
+)
+from .cairn import Cairn, cairn  # noqa: F401
+from .hash import (
+    compute_cairn_id,
+    resolve_hashable,
+)
+from cairns.patterns import rate_limited, replayable
+from .serial import (
+    Serializer,
+    from_jsonable,
+    to_jsonable,
+)
+from .sink import CompositeSink, JSONLSink, event_to_dict
+from .store import FileStore, MemoryStore, OverlayStore, Store, StoreStats
+from .types import Record, SpanMetrics, StepInfo, TaskSpan, TraceRecord
+
+__all__ = [
+    # decorator + Handle
+    "step",
+    "Handle",
+    "trace",
+    "cached_output",
+    "cached_tracing",
+    # cairn inspection
+    "Cairn",
+    "cairn",
+    # runtime
+    "Runtime",
+    "Run",
+    "default_runtime",
+    "current_run",
+    "current_span",
+    "Event",
+    "Sink",
+    "InteractionSink",
+    "MemorySink",
+    "NullSink",
+    "emit_event",
+    # hash
+    "compute_cairn_id",
+    "resolve_hashable",
+    # serial
+    "Serializer",
+    "to_jsonable",
+    "from_jsonable",
+    # sink
+    "JSONLSink",
+    "CompositeSink",
+    "event_to_dict",
+    # store
+    "Store",
+    "MemoryStore",
+    "FileStore",
+    "OverlayStore",
+    "StoreStats",
+    # patterns
+    "rate_limited",
+    "replayable",
+    # types
+    "StepInfo",
+    "TraceRecord",
+    "Record",
+    "SpanMetrics",
+    "TaskSpan",
+]

cairns/core/cairn.py

@@ -0,0 +1,98 @@
+"""The `Cairn` view + the `cairn()` accessor.
+
+A `Cairn` is a lazy, iterable view over a Store's record stack for one
+cairn_id. Newest-first iteration. Doesn't load records eagerly — yields
+on demand.
+
+Three entry points:
+
+- `cairn()` — the cairn for the currently-executing `@step`.
+- `step_fn.cairn(*args, **kwargs)` — the cairn for an arbitrary `@step`
+  invocation, computed without invoking. (Method on the decorated
+  function; defined in `step.py`.)
+- `Cairn.from_store(store, cairn_id)` — explicit construction. For CLI
+  / scripts / advanced use, when no Run is active.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Iterator
+
+from .runtime import current_run, current_span
+from .types import Record
+
+if TYPE_CHECKING:
+    from .store import Store
+
+
+class Cairn:
+    """View over the record stack for one cairn_id."""
+
+    def __init__(self, cairn_id: str, store: "Store") -> None:
+        self.cairn_id = cairn_id
+        self._store = store
+
+    @classmethod
+    def from_store(cls, store: "Store", cairn_id: str) -> "Cairn":
+        """Explicit construction. For inspection outside an active Run."""
+        return cls(cairn_id, store)
+
+    def __iter__(self) -> Iterator[Record]:
+        return self._store.iter_records(self.cairn_id)
+
+    def __len__(self) -> int:
+        return sum(1 for _ in self)
+
+    def __bool__(self) -> bool:
+        return next(iter(self), None) is not None
+
+    def latest(
+        self,
+        *,
+        version: str | None = None,
+        body_hash: str | None = None,
+        include_errors: bool = False,
+    ) -> Record | None:
+        """Newest record matching the given filters.
+
+        `version` / `body_hash` constrain on the corresponding fields when set.
+        By default skips errored records — pass `include_errors=True` to
+        surface the most recent regardless.
+        """
+        for record in self:
+            if record.error is not None and not include_errors:
+                continue
+            if version is not None and record.version != version:
+                continue
+            if body_hash is not None and record.body_hash != body_hash:
+                continue
+            return record
+        return None
+
+    def at(self, record_id: str) -> Record | None:
+        """Pinpoint a record by its id."""
+        for record in self:
+            if record.record_id == record_id:
+                return record
+        return None
+
+
+def cairn() -> Cairn:
+    """The cairn for the currently-executing `@step`.
+
+    Reads `current_span` for the step's identity + bound args, looks up
+    the cairn_id, and returns a view over the active Run's store.
+
+    Raises if no `@step` is active (i.e. called from outside a step body).
+    """
+    span = current_span.get()
+    if span is None:
+        raise RuntimeError(
+            "cairn() called outside a @step body — no current span"
+        )
+    if span.cairn_id is None:
+        raise RuntimeError(
+            "cairn() called before the step's cairn_id was computed — "
+            "this should only happen if called before _resolve_args completes"
+        )
+    return Cairn(span.cairn_id, current_run().store)

cairns/core/hash.py

@@ -0,0 +1,178 @@
+"""Hashing utilities for cache key computation.
+
+Hashers live on `Runtime` instances (see `cairns.core.runtime`). The
+default hashers (Path, functools.partial, Pydantic) are installed by
+`Runtime.__init__`. For per-Runtime customization use
+`runtime.register_hasher(...)`; for tests use `Harness(hash_funcs=...)`.
+
+`resolve_hashable` defaults to the active Run's runtime hashers, falling
+back to `default_runtime` when no Run is active (e.g. body fingerprinting
+at `@step` decoration time). Pass `hash_funcs=` explicitly to override
+for one call.
+
+Limitation: registered hashers (`_hash_partial`) that recursively call
+`resolve_hashable` will themselves see the active runtime's funcs (via
+the same default-lookup), so transitive overrides do flow through.
+"""
+
+from __future__ import annotations
+
+import functools
+import hashlib
+import json
+from pathlib import Path
+from typing import Any, Callable, cast
+
+
+def resolve_hashable(
+    value: Any,
+    _seen: dict[int, bool] | None = None,
+    hash_funcs: dict[type, Callable[[Any], Any]] | None = None,
+) -> Any:
+    """Turn any value into a canonical tree of primitives for hashing.
+
+    `hash_funcs`, if provided, fully replaces the active runtime's
+    hashers for this call (and its recursion). Otherwise the active
+    Run's runtime is consulted, falling back to `default_runtime`.
+
+    Returns a JSON-serializable structure. Unknown types raise TypeError
+    (fail-loud — no silent repr truncation of numpy/pandas/torch objects).
+    Cycles are replaced with a sentinel.
+    """
+    if _seen is None:
+        _seen = {}
+
+    if value is None or isinstance(value, (bool, int, float, str)):
+        return value
+
+    vid = id(value)
+    if vid in _seen:
+        return {"__cycle__": True}
+
+    if isinstance(value, bytes):
+        return {"__bytes__": value.hex()}
+
+    if isinstance(value, dict):
+        d = cast(dict[Any, Any], value)
+        _seen[vid] = True
+        try:
+            return {
+                "__dict__": {
+                    str(k): resolve_hashable(d[k], _seen, hash_funcs)
+                    for k in sorted(d, key=lambda x: str(x))
+                }
+            }
+        finally:
+            del _seen[vid]
+
+    if isinstance(value, (list, tuple)):
+        seq = cast(list[Any] | tuple[Any, ...], value)
+        tag = "__list__" if isinstance(value, list) else "__tuple__"
+        _seen[vid] = True
+        try:
+            return {tag: [resolve_hashable(v, _seen, hash_funcs) for v in seq]}
+        finally:
+            del _seen[vid]
+
+
+    if isinstance(value, (frozenset, set)):
+        tag = "__frozenset__" if isinstance(value, frozenset) else "__set__"
+        fs = cast(set[Any] | frozenset[Any], value)
+        _seen[vid] = True
+        try:
+            items = [resolve_hashable(v, _seen, hash_funcs) for v in fs]
+            items.sort(key=lambda x: json.dumps(x, sort_keys=True))
+            return {tag: items}
+        finally:
+            del _seen[vid]
+
+    if hash_funcs is None:
+        from .runtime import active_hash_funcs  # noqa: PLC0415
+
+        funcs = active_hash_funcs()
+    else:
+        funcs = hash_funcs
+
+    for tp in type(value).__mro__:
+        if tp in funcs:
+            return funcs[tp](value)
+
+    raise TypeError(
+        f"Unhashable type for cache key: {type(value).__name__}. "
+        f"Register a hasher via `runtime.register_hasher(...)`."
+    )
+
+
+def compute_cairn_id(identity: str, resolved_args: dict[str, Any]) -> str:
+    """Compute a cairn id: computation identity + args, excluding version."""
+    canonical = json.dumps(
+        {
+            "identity": identity,
+            "args": resolve_hashable(resolved_args),
+        },
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    return hashlib.sha256(canonical.encode()).hexdigest()
+
+
+# ── Default type hashers ──
+#
+# These are installed onto `Runtime.hash_funcs` by `_install_defaults`
+# in runtime.py. They're public-ish (exported as private-by-convention)
+# so subclassing or extending Runtimes can reuse them.
+
+
+def _hash_path(p: Path) -> Any:
+    # No resolve() — symlinks are often deliberate (pointing at a "current"
+    # artifact); resolving would invalidate on every target swap. Users
+    # wanting content-hashing or resolved paths can re-register.
+    path_str = str(p)
+    try:
+        st = p.stat()
+    except FileNotFoundError:
+        return {"__path__": {"s": path_str, "state": "missing"}}
+    except OSError as e:
+        return {"__path__": {"s": path_str, "state": "stat_error", "errno": e.errno}}
+    return {"__path__": {"s": path_str, "mtime_ns": st.st_mtime_ns, "size": st.st_size}}
+
+
+def _hash_partial(p: "functools.partial[Any]") -> Any:
+    # Reuse StepInfo.from_function so body edits to p.func invalidate. That
+    # also respects @step's attached `.info` (including user overrides).
+    from .types import StepInfo
+
+    return {
+        "__partial__": {
+            "func": StepInfo.from_function(p.func).version,
+            "args": [resolve_hashable(a) for a in p.args],
+            "keywords": {k: resolve_hashable(v) for k, v in p.keywords.items()},
+        }
+    }
+
+
+def _hash_pydantic(model: Any) -> Any:
+    # pydantic v2: model_dump(mode="json") coerces datetimes/enums/UUIDs to
+    # JSON primitives and recurses into nested models. Class qualname is
+    # included so structurally-identical models in different classes don't
+    # collide. Schema changes that don't affect dumped values (docstring edits,
+    # field reordering) don't invalidate — usually what you want.
+    cls_mod = type(model).__module__
+    cls_name = type(model).__qualname__
+    return {
+        "__pydantic__": {
+            "cls": f"{cls_mod}:{cls_name}",
+            "data": model.model_dump(mode="json"),
+        }
+    }
+
+
+def install_defaults(runtime: Any) -> None:
+    """Install Path / functools.partial / Pydantic hashers on `runtime`."""
+    runtime.hash_funcs[Path] = _hash_path
+    runtime.hash_funcs[functools.partial] = _hash_partial
+    try:
+        from pydantic import BaseModel  # noqa: PLC0415
+    except ImportError:
+        return
+    runtime.hash_funcs[BaseModel] = _hash_pydantic

cairns/core/lock.py

@@ -0,0 +1,65 @@
+"""Shared/exclusive store lock.
+
+`.cairns/gc.lock` is the rendezvous file. `FileStore.put` takes a **shared** lock
+while publishing a record (makedirs → rename); `gc_outputs` takes an **exclusive**
+lock while running mark + sweep. Multiple concurrent runs can publish; a GC
+request waits until in-flight publishes drain, and blocks new publishes until
+sweep completes.
+
+POSIX-only (fcntl). The design doc calls out that Windows isn't supported.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import os
+from contextlib import contextmanager
+from typing import Iterator
+
+
+def _lock_path(store_path: str) -> str:
+    return os.path.join(store_path, "gc.lock")
+
+
+def _ensure_lockfile(store_path: str) -> str:
+    os.makedirs(store_path, exist_ok=True)
+    path = _lock_path(store_path)
+    if not os.path.exists(path):
+        # Touch the file; O_EXCL avoids a rare race where two callers both
+        # create it. EEXIST means someone got there first — fine.
+        try:
+            fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
+            os.close(fd)
+        except FileExistsError:
+            pass
+    return path
+
+
+@contextmanager
+def store_shared(store_path: str) -> Iterator[None]:
+    """Shared (reader) lock: held by each FileStore.put during publication."""
+    path = _ensure_lockfile(store_path)
+    fd = os.open(path, os.O_RDONLY)
+    try:
+        fcntl.flock(fd, fcntl.LOCK_SH)
+        try:
+            yield
+        finally:
+            fcntl.flock(fd, fcntl.LOCK_UN)
+    finally:
+        os.close(fd)
+
+
+@contextmanager
+def gc_exclusive(store_path: str) -> Iterator[None]:
+    """Exclusive (writer) lock: held by gc during mark + sweep."""
+    path = _ensure_lockfile(store_path)
+    fd = os.open(path, os.O_RDWR)
+    try:
+        fcntl.flock(fd, fcntl.LOCK_EX)
+        try:
+            yield
+        finally:
+            fcntl.flock(fd, fcntl.LOCK_UN)
+    finally:
+        os.close(fd)