AbstractRuntime 0.2.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

abstractruntime/storage/artifacts.py

@@ -44,6 +44,7 @@ class ArtifactMetadata:
     content_type: str  # MIME type or semantic type
     size_bytes: int
     created_at: str
+    blob_id: Optional[str] = None  # Global (cross-run) content hash for dedupe
     run_id: Optional[str] = None  # Optional association with a run
     tags: Dict[str, str] = field(default_factory=dict)
 
@@ -54,6 +55,7 @@ class ArtifactMetadata:
     def from_dict(cls, data: Dict[str, Any]) -> "ArtifactMetadata":
         return cls(
             artifact_id=data["artifact_id"],
+            blob_id=data.get("blob_id"),
             content_type=data["content_type"],
             size_bytes=data["size_bytes"],
             created_at=data["created_at"],
@@ -86,9 +88,28 @@ class Artifact:
         return json.loads(self.content.decode("utf-8"))
 
 
-def compute_artifact_id(content: bytes) -> str:
-    """Compute content-addressed artifact ID using SHA-256."""
-    return hashlib.sha256(content).hexdigest()[:32]
+def compute_artifact_id(content: bytes, *, run_id: Optional[str] = None) -> str:
+    """Compute a deterministic artifact id.
+
+    By default, artifacts are content-addressed (SHA-256, truncated) so the same bytes
+    produce the same id.
+
+    If `run_id` is provided, the id is *namespaced to that run* so each run can have a
+    distinct artifact_id (while still enabling cross-run blob dedupe via `blob_id`).
+    """
+    h = hashlib.sha256()
+    if run_id is not None:
+        rid = str(run_id).strip()
+        if rid:
+            h.update(rid.encode("utf-8"))
+            h.update(b"\0")
+    h.update(content)
+    return h.hexdigest()[:32]
+
+
+def compute_blob_id(content: bytes) -> str:
+    """Compute a stable, global content hash for artifact blob dedupe."""
+    return hashlib.sha256(content).hexdigest()
 
 
 def validate_artifact_id(artifact_id: str) -> None:
@@ -318,10 +339,11 @@ class InMemoryArtifactStore(ArtifactStore):
         artifact_id: Optional[str] = None,
     ) -> ArtifactMetadata:
         if artifact_id is None:
-            artifact_id = compute_artifact_id(content)
+            artifact_id = compute_artifact_id(content, run_id=run_id)
 
         metadata = ArtifactMetadata(
             artifact_id=artifact_id,
+            blob_id=compute_blob_id(content),
             content_type=content_type,
             size_bytes=len(content),
             created_at=utc_now_iso(),
@@ -367,26 +389,77 @@ class InMemoryArtifactStore(ArtifactStore):
 class FileArtifactStore(ArtifactStore):
     """File-based artifact store.
 
-    Directory structure:
+    Directory structure (v1, cross-run blob dedupe):
+        base_dir/
+            artifacts/
+                blobs/{blob_id}.bin   # global content-addressed bytes
+                refs/{artifact_id}.meta  # per-artifact metadata (points to blob_id)
+
+    Legacy layout (v0) is still supported for reads:
         base_dir/
             artifacts/
-                {artifact_id}.bin     # content
-                {artifact_id}.meta    # metadata JSON
+                {artifact_id}.bin
+                {artifact_id}.meta
     """
 
     def __init__(self, base_dir: Union[str, Path]) -> None:
         self._base = Path(base_dir)
         self._artifacts_dir = self._base / "artifacts"
-        self._artifacts_dir.mkdir(parents=True, exist_ok=True)
+        self._blobs_dir = self._artifacts_dir / "blobs"
+        self._refs_dir = self._artifacts_dir / "refs"
+        self._blobs_dir.mkdir(parents=True, exist_ok=True)
+        self._refs_dir.mkdir(parents=True, exist_ok=True)
 
-    def _content_path(self, artifact_id: str) -> Path:
+    def _legacy_content_path(self, artifact_id: str) -> Path:
         validate_artifact_id(artifact_id)
         return self._artifacts_dir / f"{artifact_id}.bin"
 
-    def _metadata_path(self, artifact_id: str) -> Path:
+    def _legacy_metadata_path(self, artifact_id: str) -> Path:
         validate_artifact_id(artifact_id)
         return self._artifacts_dir / f"{artifact_id}.meta"
 
+    def _ref_metadata_path(self, artifact_id: str) -> Path:
+        validate_artifact_id(artifact_id)
+        return self._refs_dir / f"{artifact_id}.meta"
+
+    def _blob_path(self, blob_id: str) -> Path:
+        validate_artifact_id(blob_id)
+        return self._blobs_dir / f"{blob_id}.bin"
+
+    def _write_blob(self, *, blob_id: str, content: bytes) -> Path:
+        path = self._blob_path(blob_id)
+        if path.exists():
+            return path
+        import uuid
+
+        tmp = path.with_name(f"{path.name}.{uuid.uuid4().hex}.tmp")
+        try:
+            with open(tmp, "wb") as f:
+                f.write(content)
+            tmp.replace(path)
+        finally:
+            try:
+                if tmp.exists():
+                    tmp.unlink()
+            except Exception:
+                pass
+        return path
+
+    def _content_path(self, artifact_id: str) -> Path:
+        validate_artifact_id(artifact_id)
+        meta = self.get_metadata(artifact_id)
+        blob_id = getattr(meta, "blob_id", None) if meta is not None else None
+        if isinstance(blob_id, str) and blob_id.strip():
+            return self._blob_path(blob_id.strip())
+        return self._legacy_content_path(artifact_id)
+
+    def _metadata_path(self, artifact_id: str) -> Path:
+        validate_artifact_id(artifact_id)
+        p = self._ref_metadata_path(artifact_id)
+        if p.exists():
+            return p
+        return self._legacy_metadata_path(artifact_id)
+
     def store(
         self,
         content: bytes,
@@ -397,10 +470,12 @@ class FileArtifactStore(ArtifactStore):
         artifact_id: Optional[str] = None,
     ) -> ArtifactMetadata:
         if artifact_id is None:
-            artifact_id = compute_artifact_id(content)
+            artifact_id = compute_artifact_id(content, run_id=run_id)
+        blob_id = compute_blob_id(content)
 
         metadata = ArtifactMetadata(
             artifact_id=artifact_id,
+            blob_id=blob_id,
             content_type=content_type,
             size_bytes=len(content),
             created_at=utc_now_iso(),
@@ -408,39 +483,40 @@ class FileArtifactStore(ArtifactStore):
             tags=tags or {},
         )
 
-        # Write content
-        content_path = self._content_path(artifact_id)
-        with open(content_path, "wb") as f:
-            f.write(content)
+        # Write blob bytes (deduped across runs)
+        self._write_blob(blob_id=blob_id, content=content)
 
         # Write metadata
-        metadata_path = self._metadata_path(artifact_id)
+        metadata_path = self._ref_metadata_path(artifact_id)
         with open(metadata_path, "w", encoding="utf-8") as f:
             json.dump(metadata.to_dict(), f, ensure_ascii=False, indent=2)
 
         return metadata
 
     def load(self, artifact_id: str) -> Optional[Artifact]:
-        content_path = self._content_path(artifact_id)
         metadata_path = self._metadata_path(artifact_id)
-
-        if not content_path.exists() or not metadata_path.exists():
+        if not metadata_path.exists():
             return None
 
-        with open(content_path, "rb") as f:
-            content = f.read()
-
         with open(metadata_path, "r", encoding="utf-8") as f:
             metadata_dict = json.load(f)
 
         metadata = ArtifactMetadata.from_dict(metadata_dict)
+        content_path = self._content_path(artifact_id)
+        if not content_path.exists():
+            return None
+
+        with open(content_path, "rb") as f:
+            content = f.read()
         return Artifact(metadata=metadata, content=content)
 
     def get_metadata(self, artifact_id: str) -> Optional[ArtifactMetadata]:
-        metadata_path = self._metadata_path(artifact_id)
-
+        validate_artifact_id(artifact_id)
+        metadata_path = self._ref_metadata_path(artifact_id)
         if not metadata_path.exists():
-            return None
+            metadata_path = self._legacy_metadata_path(artifact_id)
+            if not metadata_path.exists():
+                return None
 
         with open(metadata_path, "r", encoding="utf-8") as f:
             metadata_dict = json.load(f)
@@ -448,28 +524,42 @@ class FileArtifactStore(ArtifactStore):
         return ArtifactMetadata.from_dict(metadata_dict)
 
     def exists(self, artifact_id: str) -> bool:
+        meta = self.get_metadata(artifact_id)
+        if meta is None:
+            return False
         return self._content_path(artifact_id).exists()
 
     def delete(self, artifact_id: str) -> bool:
-        content_path = self._content_path(artifact_id)
-        metadata_path = self._metadata_path(artifact_id)
+        validate_artifact_id(artifact_id)
+        metadata_path = self._ref_metadata_path(artifact_id)
+        legacy_meta = self._legacy_metadata_path(artifact_id)
+        legacy_content = self._legacy_content_path(artifact_id)
 
         deleted = False
-        if content_path.exists():
-            content_path.unlink()
-            deleted = True
         if metadata_path.exists():
             metadata_path.unlink()
             deleted = True
+        if legacy_meta.exists():
+            legacy_meta.unlink()
+            deleted = True
+        if legacy_content.exists():
+            legacy_content.unlink()
+            deleted = True
 
         return deleted
 
     def list_by_run(self, run_id: str) -> List[ArtifactMetadata]:
         results = []
-        for metadata_path in self._artifacts_dir.glob("*.meta"):
+        meta_paths = list(self._refs_dir.glob("*.meta")) + list(self._artifacts_dir.glob("*.meta"))
+        seen: set[str] = set()
+        for metadata_path in meta_paths:
             try:
                 with open(metadata_path, "r", encoding="utf-8") as f:
                     metadata_dict = json.load(f)
+                artifact_id = str(metadata_dict.get("artifact_id") or "").strip()
+                if not artifact_id or artifact_id in seen:
+                    continue
+                seen.add(artifact_id)
                 if metadata_dict.get("run_id") == run_id:
                     results.append(ArtifactMetadata.from_dict(metadata_dict))
             except (json.JSONDecodeError, IOError):
@@ -478,10 +568,16 @@ class FileArtifactStore(ArtifactStore):
 
     def list_all(self, *, limit: int = 1000) -> List[ArtifactMetadata]:
         results = []
-        for metadata_path in self._artifacts_dir.glob("*.meta"):
+        meta_paths = list(self._refs_dir.glob("*.meta")) + list(self._artifacts_dir.glob("*.meta"))
+        seen: set[str] = set()
+        for metadata_path in meta_paths:
             try:
                 with open(metadata_path, "r", encoding="utf-8") as f:
                     metadata_dict = json.load(f)
+                artifact_id = str(metadata_dict.get("artifact_id") or "").strip()
+                if not artifact_id or artifact_id in seen:
+                    continue
+                seen.add(artifact_id)
                 results.append(ArtifactMetadata.from_dict(metadata_dict))
             except (json.JSONDecodeError, IOError):
                 continue
@@ -489,6 +585,53 @@ class FileArtifactStore(ArtifactStore):
         results.sort(key=lambda m: m.created_at, reverse=True)
         return results[:limit]
 
+    def gc(self, *, dry_run: bool = True) -> Dict[str, Any]:
+        """Garbage collect unreferenced blobs.
+
+        Notes:
+        - This only applies to the v1 `artifacts/blobs` layout.
+        - Safe-by-default: `dry_run=True` returns the plan without deleting.
+        """
+
+        report: Dict[str, Any] = {
+            "dry_run": bool(dry_run),
+            "blobs_total": 0,
+            "blobs_referenced": 0,
+            "blobs_deleted": 0,
+            "bytes_reclaimed": 0,
+            "errors": [],
+        }
+
+        referenced: set[str] = set()
+        for meta in self.list_all(limit=1_000_000):
+            blob_id = getattr(meta, "blob_id", None)
+            if isinstance(blob_id, str) and blob_id.strip():
+                referenced.add(blob_id.strip())
+
+        report["blobs_referenced"] = len(referenced)
+
+        blobs = list(self._blobs_dir.glob("*.bin"))
+        report["blobs_total"] = len(blobs)
+
+        for p in blobs:
+            blob_id = p.stem
+            if blob_id in referenced:
+                continue
+            try:
+                size = p.stat().st_size
+            except Exception:
+                size = 0
+            if not dry_run:
+                try:
+                    p.unlink()
+                except Exception as e:
+                    report["errors"].append({"blob_id": blob_id, "error": str(e)})
+                    continue
+            report["blobs_deleted"] += 1
+            report["bytes_reclaimed"] += int(size)
+
+        return report
+
 
 # Artifact reference helpers for use in RunState.vars
 

abstractruntime/storage/base.py

@@ -95,6 +95,23 @@ class QueryableRunStore(Protocol):
         ...
 
 
+@runtime_checkable
+class QueryableRunIndexStore(Protocol):
+    """Optional fast-path for listing run summaries without loading full RunState payloads."""
+
+    def list_run_index(
+        self,
+        *,
+        status: Optional[RunStatus] = None,
+        workflow_id: Optional[str] = None,
+        session_id: Optional[str] = None,
+        root_only: bool = False,
+        limit: int = 100,
+    ) -> List[Dict[str, Any]]:
+        """List lightweight run index rows (most recent first)."""
+        ...
+
+
 class LedgerStore(ABC):
     """Append-only journal store."""
 
@@ -104,4 +121,3 @@ class LedgerStore(ABC):
     @abstractmethod
     def list(self, run_id: str) -> List[Dict[str, Any]]: ...
 
-

abstractruntime/storage/commands.py

@@ -0,0 +1,339 @@
+"""abstractruntime.storage.commands
+
+Durable command inbox primitives (append-only, idempotent).
+
+Why this exists:
+- A remote Run Gateway (ADR-0018 / backlog 307) needs a control plane that is safe under
+  retries and intermittent networks.
+- The key SQS/Temporal insight is to decouple *command acceptance* from *fulfillment*:
+  clients submit commands with idempotency keys, and a worker processes them asynchronously.
+
+Design constraints:
+- JSON-safe records only (persisted).
+- Append-only storage (audit-friendly, replayable).
+- Idempotency by `command_id` (duplicate submissions are ignored).
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import uuid
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Protocol, Tuple, runtime_checkable
+
+
+def _utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _is_json_value(value: Any) -> bool:
+    if value is None or isinstance(value, (str, int, float, bool)):
+        return True
+    if isinstance(value, list):
+        return all(_is_json_value(v) for v in value)
+    if isinstance(value, dict):
+        return all(isinstance(k, str) and _is_json_value(v) for k, v in value.items())
+    return False
+
+
+@dataclass(frozen=True)
+class CommandRecord:
+    """A durable command record.
+
+    Notes:
+    - `seq` is assigned by the store and provides cursor semantics for consumers.
+    - `payload` must be JSON-serializable (dict of JSON values).
+    """
+
+    command_id: str
+    run_id: str
+    type: str
+    payload: Dict[str, Any]
+    ts: str
+    client_id: Optional[str] = None
+    seq: int = 0
+
+    def to_json(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class CommandAppendResult:
+    """Result of appending a command to a CommandStore."""
+
+    accepted: bool
+    duplicate: bool
+    seq: int
+
+
+@runtime_checkable
+class CommandStore(Protocol):
+    """Append-only inbox of commands with cursor replay semantics."""
+
+    def append(self, record: CommandRecord) -> CommandAppendResult:
+        """Append a command if its command_id is new (idempotent)."""
+
+    def list_after(self, *, after: int, limit: int = 1000) -> Tuple[List[CommandRecord], int]:
+        """Return commands with seq > after, up to limit, and the next cursor."""
+
+    def get_last_seq(self) -> int:
+        """Return the greatest assigned sequence number (0 if empty)."""
+
+
+@runtime_checkable
+class CommandCursorStore(Protocol):
+    """Durable consumer cursor for CommandStore replay."""
+
+    def load(self) -> int: ...
+
+    def save(self, cursor: int) -> None: ...
+
+
+class InMemoryCommandStore(CommandStore):
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._seq = 0
+        self._by_id: Dict[str, CommandRecord] = {}
+        self._ordered: List[CommandRecord] = []
+
+    def append(self, record: CommandRecord) -> CommandAppendResult:
+        cid = str(record.command_id or "").strip()
+        if not cid:
+            cid = uuid.uuid4().hex
+        with self._lock:
+            existing = self._by_id.get(cid)
+            if existing is not None:
+                return CommandAppendResult(accepted=False, duplicate=True, seq=int(existing.seq or 0))
+            self._seq += 1
+            rec = CommandRecord(
+                command_id=cid,
+                run_id=str(record.run_id or ""),
+                type=str(record.type or ""),
+                payload=dict(record.payload or {}),
+                ts=str(record.ts or _utc_now_iso()),
+                client_id=str(record.client_id) if isinstance(record.client_id, str) and record.client_id else None,
+                seq=self._seq,
+            )
+            self._by_id[cid] = rec
+            self._ordered.append(rec)
+            return CommandAppendResult(accepted=True, duplicate=False, seq=rec.seq)
+
+    def list_after(self, *, after: int, limit: int = 1000) -> Tuple[List[CommandRecord], int]:
+        after2 = int(after or 0)
+        limit2 = int(limit or 1000)
+        if limit2 <= 0:
+            limit2 = 1000
+        with self._lock:
+            items = [r for r in self._ordered if int(r.seq or 0) > after2]
+            out = items[:limit2]
+            next_cursor = after2
+            if out:
+                next_cursor = int(out[-1].seq or after2)
+            return (list(out), next_cursor)
+
+    def get_last_seq(self) -> int:
+        with self._lock:
+            return int(self._seq or 0)
+
+
+class InMemoryCommandCursorStore(CommandCursorStore):
+    def __init__(self, initial: int = 0) -> None:
+        self._cursor = int(initial or 0)
+        self._lock = threading.Lock()
+
+    def load(self) -> int:
+        with self._lock:
+            return int(self._cursor or 0)
+
+    def save(self, cursor: int) -> None:
+        with self._lock:
+            self._cursor = int(cursor or 0)
+
+
+class JsonFileCommandCursorStore(CommandCursorStore):
+    """JSON file-backed cursor store.
+
+    Atomic write semantics are important because this file is updated frequently.
+    """
+
+    def __init__(self, path: str | Path) -> None:
+        self._path = Path(path)
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.Lock()
+
+    def load(self) -> int:
+        with self._lock:
+            if not self._path.exists():
+                return 0
+            try:
+                data = json.loads(self._path.read_text(encoding="utf-8") or "{}")
+            except Exception:
+                return 0
+            cur = data.get("cursor")
+            try:
+                return int(cur or 0)
+            except Exception:
+                return 0
+
+    def save(self, cursor: int) -> None:
+        cur = int(cursor or 0)
+        tmp = self._path.with_name(f"{self._path.name}.{uuid.uuid4().hex}.tmp")
+        payload = {"cursor": cur, "updated_at": _utc_now_iso()}
+        with self._lock:
+            try:
+                tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+                tmp.replace(self._path)
+            finally:
+                try:
+                    if tmp.exists():
+                        tmp.unlink()
+                except Exception:
+                    pass
+
+
+class JsonlCommandStore(CommandStore):
+    """Append-only JSONL command store.
+
+    File format: one JSON object per line. Each record includes a store-assigned `seq`.
+    """
+
+    def __init__(self, base_dir: str | Path, *, filename: str = "commands.jsonl") -> None:
+        self._base = Path(base_dir)
+        self._base.mkdir(parents=True, exist_ok=True)
+        self._path = self._base / str(filename or "commands.jsonl")
+        self._lock = threading.Lock()
+
+        # Idempotency index (rebuilt on init from the log).
+        self._seq = 0
+        self._by_id: Dict[str, int] = {}
+        self._rebuild_index()
+
+    def _rebuild_index(self) -> None:
+        if not self._path.exists():
+            self._seq = 0
+            self._by_id = {}
+            return
+        seq = 0
+        by_id: Dict[str, int] = {}
+        try:
+            with self._path.open("r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        obj = json.loads(line)
+                    except Exception:
+                        continue
+                    s = obj.get("seq")
+                    cid = obj.get("command_id")
+                    try:
+                        s_int = int(s or 0)
+                    except Exception:
+                        s_int = 0
+                    if s_int <= 0:
+                        continue
+                    if s_int > seq:
+                        seq = s_int
+                    if isinstance(cid, str) and cid and cid not in by_id:
+                        by_id[cid] = s_int
+        except Exception:
+            seq = 0
+            by_id = {}
+        self._seq = seq
+        self._by_id = by_id
+
+    def append(self, record: CommandRecord) -> CommandAppendResult:
+        cid = str(record.command_id or "").strip()
+        if not cid:
+            cid = uuid.uuid4().hex
+        run_id = str(record.run_id or "").strip()
+        typ = str(record.type or "").strip()
+        payload = dict(record.payload or {})
+        ts = str(record.ts or "").strip() or _utc_now_iso()
+        client_id = str(record.client_id).strip() if isinstance(record.client_id, str) and record.client_id else None
+
+        if not run_id:
+            raise ValueError("CommandRecord.run_id must be non-empty")
+        if not typ:
+            raise ValueError("CommandRecord.type must be non-empty")
+        if not isinstance(payload, dict) or not _is_json_value(payload):
+            raise ValueError("CommandRecord.payload must be a JSON-serializable dict")
+
+        with self._lock:
+            existing_seq = self._by_id.get(cid)
+            if existing_seq is not None:
+                return CommandAppendResult(accepted=False, duplicate=True, seq=int(existing_seq))
+
+            self._seq += 1
+            seq = int(self._seq)
+            rec = CommandRecord(
+                command_id=cid,
+                run_id=run_id,
+                type=typ,
+                payload=payload,
+                ts=ts,
+                client_id=client_id,
+                seq=seq,
+            )
+            with self._path.open("a", encoding="utf-8") as f:
+                f.write(json.dumps(rec.to_json(), ensure_ascii=False))
+                f.write("\n")
+            self._by_id[cid] = seq
+            return CommandAppendResult(accepted=True, duplicate=False, seq=seq)
+
+    def list_after(self, *, after: int, limit: int = 1000) -> Tuple[List[CommandRecord], int]:
+        after2 = int(after or 0)
+        limit2 = int(limit or 1000)
+        if limit2 <= 0:
+            limit2 = 1000
+
+        if not self._path.exists():
+            return ([], after2)
+
+        out: List[CommandRecord] = []
+        next_cursor = after2
+        try:
+            with self._path.open("r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        obj = json.loads(line)
+                    except Exception:
+                        continue
+                    try:
+                        seq = int(obj.get("seq") or 0)
+                    except Exception:
+                        continue
+                    if seq <= after2:
+                        continue
+                    try:
+                        rec = CommandRecord(
+                            command_id=str(obj.get("command_id") or ""),
+                            run_id=str(obj.get("run_id") or ""),
+                            type=str(obj.get("type") or ""),
+                            payload=dict(obj.get("payload") or {}),
+                            ts=str(obj.get("ts") or ""),
+                            client_id=str(obj.get("client_id") or "") or None,
+                            seq=seq,
+                        )
+                    except Exception:
+                        continue
+                    out.append(rec)
+                    next_cursor = seq
+                    if len(out) >= limit2:
+                        break
+        except Exception:
+            return ([], after2)
+        return (out, next_cursor)
+
+    def get_last_seq(self) -> int:
+        with self._lock:
+            return int(self._seq or 0)
+
+