AbstractRuntime 0.0.0__py3-none-any.whl → 0.0.1__py3-none-any.whl

abstractruntime/storage/json_files.py

@@ -0,0 +1,208 @@
+"""abstractruntime.storage.json_files
+
+Simple file-based persistence:
+- RunState checkpoints as JSON (one file per run)
+- Ledger as JSONL (append-only)
+
+This is meant as a straightforward MVP backend.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from .base import LedgerStore, RunStore
+from ..core.models import RunState, StepRecord, RunStatus, WaitState, WaitReason
+
+
+class JsonFileRunStore(RunStore):
+    """File-based run store with query support.
+
+    Implements both RunStore (ABC) and QueryableRunStore (Protocol).
+
+    Query operations scan all run_*.json files, which is acceptable for MVP
+    but may need indexing for large deployments.
+    """
+
+    def __init__(self, base_dir: str | Path):
+        self._base = Path(base_dir)
+        self._base.mkdir(parents=True, exist_ok=True)
+
+    def _path(self, run_id: str) -> Path:
+        return self._base / f"run_{run_id}.json"
+
+    def save(self, run: RunState) -> None:
+        p = self._path(run.run_id)
+        with p.open("w", encoding="utf-8") as f:
+            json.dump(asdict(run), f, ensure_ascii=False, indent=2)
+
+    def load(self, run_id: str) -> Optional[RunState]:
+        p = self._path(run_id)
+        if not p.exists():
+            return None
+        return self._load_from_path(p)
+
+    def _load_from_path(self, p: Path) -> Optional[RunState]:
+        """Load a RunState from a file path."""
+        try:
+            with p.open("r", encoding="utf-8") as f:
+                data = json.load(f)
+        except (json.JSONDecodeError, IOError):
+            return None
+
+        # Reconstruct enums and nested dataclasses
+        raw_status = data.get("status")
+        status = raw_status if isinstance(raw_status, RunStatus) else RunStatus(str(raw_status))
+
+        waiting: Optional[WaitState] = None
+        raw_waiting = data.get("waiting")
+        if isinstance(raw_waiting, dict):
+            raw_reason = raw_waiting.get("reason")
+            if raw_reason is None:
+                raise ValueError("Persisted waiting state missing 'reason'")
+            reason = raw_reason if isinstance(raw_reason, WaitReason) else WaitReason(str(raw_reason))
+            waiting = WaitState(
+                reason=reason,
+                wait_key=raw_waiting.get("wait_key"),
+                until=raw_waiting.get("until"),
+                resume_to_node=raw_waiting.get("resume_to_node"),
+                result_key=raw_waiting.get("result_key"),
+                prompt=raw_waiting.get("prompt"),
+                choices=raw_waiting.get("choices"),
+                allow_free_text=bool(raw_waiting.get("allow_free_text", True)),
+                details=raw_waiting.get("details"),
+            )
+
+        return RunState(
+            run_id=data["run_id"],
+            workflow_id=data["workflow_id"],
+            status=status,
+            current_node=data["current_node"],
+            vars=data.get("vars") or {},
+            waiting=waiting,
+            output=data.get("output"),
+            error=data.get("error"),
+            created_at=data.get("created_at"),
+            updated_at=data.get("updated_at"),
+            actor_id=data.get("actor_id"),
+            parent_run_id=data.get("parent_run_id"),
+        )
+
+    def _iter_all_runs(self) -> List[RunState]:
+        """Iterate over all stored runs."""
+        runs: List[RunState] = []
+        for p in self._base.glob("run_*.json"):
+            run = self._load_from_path(p)
+            if run is not None:
+                runs.append(run)
+        return runs
+
+    # --- QueryableRunStore methods ---
+
+    def list_runs(
+        self,
+        *,
+        status: Optional[RunStatus] = None,
+        wait_reason: Optional[WaitReason] = None,
+        workflow_id: Optional[str] = None,
+        limit: int = 100,
+    ) -> List[RunState]:
+        """List runs matching the given filters."""
+        results: List[RunState] = []
+
+        for run in self._iter_all_runs():
+            # Apply filters
+            if status is not None and run.status != status:
+                continue
+            if workflow_id is not None and run.workflow_id != workflow_id:
+                continue
+            if wait_reason is not None:
+                if run.waiting is None or run.waiting.reason != wait_reason:
+                    continue
+
+            results.append(run)
+
+        # Sort by updated_at descending (most recent first)
+        results.sort(key=lambda r: r.updated_at or "", reverse=True)
+
+        return results[:limit]
+
+    def list_due_wait_until(
+        self,
+        *,
+        now_iso: str,
+        limit: int = 100,
+    ) -> List[RunState]:
+        """List runs waiting for a time threshold that has passed."""
+        results: List[RunState] = []
+
+        for run in self._iter_all_runs():
+            # Must be WAITING with reason UNTIL
+            if run.status != RunStatus.WAITING:
+                continue
+            if run.waiting is None:
+                continue
+            if run.waiting.reason != WaitReason.UNTIL:
+                continue
+            if run.waiting.until is None:
+                continue
+
+            # Check if the wait time has passed (ISO string comparison works for UTC)
+            if run.waiting.until <= now_iso:
+                results.append(run)
+
+        # Sort by waiting.until ascending (earliest due first)
+        results.sort(key=lambda r: r.waiting.until if r.waiting else "")
+
+        return results[:limit]
+
+    def list_children(
+        self,
+        *,
+        parent_run_id: str,
+        status: Optional[RunStatus] = None,
+    ) -> List[RunState]:
+        """List child runs of a parent."""
+        results: List[RunState] = []
+
+        for run in self._iter_all_runs():
+            if run.parent_run_id != parent_run_id:
+                continue
+            if status is not None and run.status != status:
+                continue
+            results.append(run)
+
+        return results
+
+
+class JsonlLedgerStore(LedgerStore):
+    def __init__(self, base_dir: str | Path):
+        self._base = Path(base_dir)
+        self._base.mkdir(parents=True, exist_ok=True)
+
+    def _path(self, run_id: str) -> Path:
+        return self._base / f"ledger_{run_id}.jsonl"
+
+    def append(self, record: StepRecord) -> None:
+        p = self._path(record.run_id)
+        with p.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(asdict(record), ensure_ascii=False))
+            f.write("\n")
+
+    def list(self, run_id: str) -> List[Dict[str, Any]]:
+        p = self._path(run_id)
+        if not p.exists():
+            return []
+        out: List[Dict[str, Any]] = []
+        with p.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                out.append(json.loads(line))
+        return out
+
+

abstractruntime/storage/ledger_chain.py

@@ -0,0 +1,153 @@
+"""abstractruntime.storage.ledger_chain
+
+Tamper-evident provenance for the execution ledger.
+
+This module provides:
+- A `HashChainedLedgerStore` decorator that wraps any `LedgerStore` and injects
+  `prev_hash` + `record_hash` into each appended `StepRecord`.
+- A `verify_ledger_chain()` utility to validate the chain.
+
+Important scope boundary:
+- This is **tamper-evident**, not tamper-proof.
+- Cryptographic signatures (non-forgeability) are intentionally out of scope for v0.1.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict
+from typing import Any, Dict, List, Optional
+
+from .base import LedgerStore
+from ..core.models import StepRecord
+
+
+def _canonical_json(data: Dict[str, Any]) -> str:
+    return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+
+
+def _sha256_hex(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def compute_record_hash(*, record: Dict[str, Any], prev_hash: Optional[str]) -> str:
+    """Compute record hash from a JSON dict.
+
+    Rules:
+    - `prev_hash` is included in the hashed payload.
+    - `record_hash` and `signature` fields are excluded (to avoid recursion).
+    """
+
+    clean = dict(record)
+    clean.pop("record_hash", None)
+    clean.pop("signature", None)
+
+    clean["prev_hash"] = prev_hash
+    return _sha256_hex(_canonical_json(clean))
+
+
+class HashChainedLedgerStore(LedgerStore):
+    """LedgerStore decorator adding a SHA-256 hash chain."""
+
+    def __init__(self, inner: LedgerStore):
+        self._inner = inner
+        self._head_by_run: Dict[str, Optional[str]] = {}
+
+    def _get_head(self, run_id: str) -> Optional[str]:
+        if run_id in self._head_by_run:
+            return self._head_by_run[run_id]
+
+        # Best-effort bootstrap for process restarts.
+        records = self._inner.list(run_id)
+        if not records:
+            self._head_by_run[run_id] = None
+            return None
+
+        last = records[-1]
+        head = last.get("record_hash")
+        self._head_by_run[run_id] = head
+        return head
+
+    def append(self, record: StepRecord) -> None:
+        prev = self._get_head(record.run_id)
+
+        # Compute hash from record dict + prev hash.
+        record.prev_hash = prev
+        record_dict = asdict(record)
+        record_hash = compute_record_hash(record=record_dict, prev_hash=prev)
+        record.record_hash = record_hash
+
+        self._inner.append(record)
+        self._head_by_run[record.run_id] = record_hash
+
+    def list(self, run_id: str) -> List[Dict[str, Any]]:
+        return self._inner.list(run_id)
+
+
+def verify_ledger_chain(records: List[Dict[str, Any]]) -> Dict[str, Any]:
+    """Verify a list of stored ledger records.
+
+    Returns a structured report with the first failing index and error details.
+    """
+
+    report: Dict[str, Any] = {
+        "ok": True,
+        "count": len(records),
+        "errors": [],
+        "first_bad_index": None,
+        "head_hash": records[-1].get("record_hash") if records else None,
+        "computed_head_hash": None,
+    }
+
+    prev: Optional[str] = None
+    computed_head: Optional[str] = None
+
+    for i, r in enumerate(records):
+        expected_prev = prev
+        actual_prev = r.get("prev_hash")
+
+        if actual_prev != expected_prev:
+            report["ok"] = False
+            report["first_bad_index"] = report["first_bad_index"] or i
+            report["errors"].append(
+                {
+                    "index": i,
+                    "type": "prev_hash_mismatch",
+                    "expected_prev_hash": expected_prev,
+                    "actual_prev_hash": actual_prev,
+                }
+            )
+
+        stored_hash = r.get("record_hash")
+        if not stored_hash:
+            report["ok"] = False
+            report["first_bad_index"] = report["first_bad_index"] or i
+            report["errors"].append(
+                {
+                    "index": i,
+                    "type": "missing_record_hash",
+                }
+            )
+            # Cannot continue computing chain reliably
+            break
+
+        computed_hash = compute_record_hash(record=r, prev_hash=actual_prev)
+        if computed_hash != stored_hash:
+            report["ok"] = False
+            report["first_bad_index"] = report["first_bad_index"] or i
+            report["errors"].append(
+                {
+                    "index": i,
+                    "type": "record_hash_mismatch",
+                    "stored": stored_hash,
+                    "computed": computed_hash,
+                }
+            )
+
+        prev = stored_hash
+        computed_head = stored_hash
+
+    report["computed_head_hash"] = computed_head
+    return report
+

abstractruntime/storage/snapshots.py

@@ -0,0 +1,217 @@
+"""abstractruntime.storage.snapshots
+
+Named snapshots/bookmarks for durable run state.
+
+A snapshot is a *user/operator-facing checkpoint* with:
+- name/description/tags
+- timestamps
+- embedded (JSON) run state
+
+This is intentionally simple for v0.1 (file-per-snapshot).
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from ..core.models import RunState
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+@dataclass
+class Snapshot:
+    snapshot_id: str
+    run_id: str
+
+    step_id: Optional[str] = None
+
+    name: str = ""
+    description: Optional[str] = None
+    tags: List[str] = field(default_factory=list)
+
+    created_at: str = field(default_factory=utc_now_iso)
+    updated_at: str = field(default_factory=utc_now_iso)
+
+    # JSON-only representation of the run state
+    run_state: Dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_run(
+        cls,
+        *,
+        run: RunState,
+        name: str,
+        description: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        step_id: Optional[str] = None,
+    ) -> "Snapshot":
+        return cls(
+            snapshot_id=str(uuid.uuid4()),
+            run_id=run.run_id,
+            step_id=step_id,
+            name=name,
+            description=description,
+            tags=list(tags or []),
+            run_state=asdict(run),
+        )
+
+
+class SnapshotStore(ABC):
+    @abstractmethod
+    def save(self, snapshot: Snapshot) -> None: ...
+
+    @abstractmethod
+    def load(self, snapshot_id: str) -> Optional[Snapshot]: ...
+
+    @abstractmethod
+    def delete(self, snapshot_id: str) -> bool: ...
+
+    @abstractmethod
+    def list(
+        self,
+        *,
+        run_id: Optional[str] = None,
+        tag: Optional[str] = None,
+        query: Optional[str] = None,
+        limit: int = 100,
+    ) -> List[Snapshot]: ...
+
+
+class InMemorySnapshotStore(SnapshotStore):
+    def __init__(self):
+        self._snaps: Dict[str, Snapshot] = {}
+
+    def save(self, snapshot: Snapshot) -> None:
+        snapshot.updated_at = utc_now_iso()
+        self._snaps[snapshot.snapshot_id] = snapshot
+
+    def load(self, snapshot_id: str) -> Optional[Snapshot]:
+        return self._snaps.get(snapshot_id)
+
+    def delete(self, snapshot_id: str) -> bool:
+        return self._snaps.pop(snapshot_id, None) is not None
+
+    def list(
+        self,
+        *,
+        run_id: Optional[str] = None,
+        tag: Optional[str] = None,
+        query: Optional[str] = None,
+        limit: int = 100,
+    ) -> List[Snapshot]:
+        snaps = list(self._snaps.values())
+        return _filter_snapshots(snaps, run_id=run_id, tag=tag, query=query, limit=limit)
+
+
+class JsonSnapshotStore(SnapshotStore):
+    """File-based snapshot store (one JSON file per snapshot)."""
+
+    def __init__(self, base_dir: str | Path):
+        self._base = Path(base_dir)
+        self._base.mkdir(parents=True, exist_ok=True)
+
+    def _path(self, snapshot_id: str) -> Path:
+        return self._base / f"snapshot_{snapshot_id}.json"
+
+    def save(self, snapshot: Snapshot) -> None:
+        snapshot.updated_at = utc_now_iso()
+        p = self._path(snapshot.snapshot_id)
+        with p.open("w", encoding="utf-8") as f:
+            json.dump(asdict(snapshot), f, ensure_ascii=False, indent=2)
+
+    def load(self, snapshot_id: str) -> Optional[Snapshot]:
+        p = self._path(snapshot_id)
+        if not p.exists():
+            return None
+        with p.open("r", encoding="utf-8") as f:
+            data = json.load(f)
+        return Snapshot(
+            snapshot_id=data["snapshot_id"],
+            run_id=data["run_id"],
+            step_id=data.get("step_id"),
+            name=data.get("name") or "",
+            description=data.get("description"),
+            tags=list(data.get("tags") or []),
+            created_at=data.get("created_at") or utc_now_iso(),
+            updated_at=data.get("updated_at") or utc_now_iso(),
+            run_state=dict(data.get("run_state") or {}),
+        )
+
+    def delete(self, snapshot_id: str) -> bool:
+        p = self._path(snapshot_id)
+        if not p.exists():
+            return False
+        p.unlink()
+        return True
+
+    def list(
+        self,
+        *,
+        run_id: Optional[str] = None,
+        tag: Optional[str] = None,
+        query: Optional[str] = None,
+        limit: int = 100,
+    ) -> List[Snapshot]:
+        snaps: List[Snapshot] = []
+        for p in sorted(self._base.glob("snapshot_*.json")):
+            try:
+                with p.open("r", encoding="utf-8") as f:
+                    data = json.load(f)
+                snaps.append(
+                    Snapshot(
+                        snapshot_id=data["snapshot_id"],
+                        run_id=data["run_id"],
+                        step_id=data.get("step_id"),
+                        name=data.get("name") or "",
+                        description=data.get("description"),
+                        tags=list(data.get("tags") or []),
+                        created_at=data.get("created_at") or utc_now_iso(),
+                        updated_at=data.get("updated_at") or utc_now_iso(),
+                        run_state=dict(data.get("run_state") or {}),
+                    )
+                )
+            except Exception:
+                # Corrupt snapshot files are ignored for now (MVP); verification tools can be added later.
+                continue
+
+        return _filter_snapshots(snaps, run_id=run_id, tag=tag, query=query, limit=limit)
+
+
+def _filter_snapshots(
+    snaps: List[Snapshot],
+    *,
+    run_id: Optional[str],
+    tag: Optional[str],
+    query: Optional[str],
+    limit: int,
+) -> List[Snapshot]:
+    out = snaps
+
+    if run_id:
+        out = [s for s in out if s.run_id == run_id]
+
+    if tag:
+        tag_l = tag.lower()
+        out = [s for s in out if any(t.lower() == tag_l for t in (s.tags or []))]
+
+    if query:
+        q = query.lower()
+        def _matches(s: Snapshot) -> bool:
+            name = (s.name or "").lower()
+            desc = (s.description or "").lower()
+            return q in name or q in desc
+        out = [s for s in out if _matches(s)]
+
+    # Newest first (best-effort ISO ordering)
+    out.sort(key=lambda s: s.created_at or "", reverse=True)
+    return out[: max(0, int(limit))]
+