conceptkernel 1.0.0__py3-none-any.whl

cklib/prov.py

@@ -0,0 +1,631 @@
+"""PROV-O session and provenance for any Concept Kernel.
+
+Generic provenance recording — no domain-specific code. Any kernel can
+import ProvChain to track sessions, record actions with before/after state,
+and run verified-action loops (execute -> verify -> record -> retry).
+
+Storage: storage/sessions/{session_id}.json (DATA loop)
+Retention: configurable via retention_hours (default 48)
+
+Usage:
+    from cklib.prov import ProvChain, verified_action
+
+    prov = ProvChain(
+        storage_dir="concepts/MyKernel",
+        label="deploy-widgets",
+        kernel_urn="ckp://Kernel#MyKernel:v1.0",
+    )
+
+    prov.record_action(
+        action_type="widget.deploy",
+        outcome="verified",
+        kernel="MyKernel",
+        intent={"target": "prod"},
+        evidence={"deployed": True},
+    )
+
+    verified_action(
+        action_fn=lambda: do_deploy(),
+        verify_fn=lambda: check_deployed(),
+        prov=prov,
+        action_type="widget.deploy",
+        kernel="MyKernel",
+    )
+
+    prov.close()
+"""
+
+import glob
+import json
+import os
+import sys
+import time as _time
+import uuid
+from dataclasses import dataclass, field, asdict
+from datetime import datetime, timedelta, timezone
+from typing import Callable
+
+from cklib.urn import build_action_urn, build_kernel_urn
+
+__all__ = [
+    "ProvChain",
+    "Session",
+    "ActionRecord",
+    "verified_action",
+    "list_sessions",
+    "get_session",
+]
+
+
+# ---------------------------------------------------------------------------
+# JSON serialisation helper — handles datetime, sets, dataclass fallback
+# ---------------------------------------------------------------------------
+
+def _json_default(obj):
+    """Custom JSON serialiser for dataclass fields."""
+    if isinstance(obj, datetime):
+        return obj.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    if isinstance(obj, set):
+        return sorted(obj)
+    raise TypeError("Object of type %s is not JSON serializable" % type(obj).__name__)
+
+
+def _to_dict(obj):
+    """Convert a dataclass to a JSON-safe dict, stripping None values."""
+    raw = asdict(obj)
+    return _strip_none(raw)
+
+
+def _strip_none(d):
+    """Recursively remove None-valued keys from a dict."""
+    if isinstance(d, dict):
+        return {k: _strip_none(v) for k, v in d.items() if v is not None}
+    if isinstance(d, list):
+        return [_strip_none(v) for v in d]
+    return d
+
+
+# ---------------------------------------------------------------------------
+# ActionRecord
+# ---------------------------------------------------------------------------
+
+@dataclass
+class ActionRecord:
+    """A single recorded action within a provenance session.
+
+    Fields follow PROV-O: the action was generated by a kernel,
+    optionally via an edge predicate (COMPOSES / EXTENDS / REQUIRES).
+    intent captures what the action wanted; evidence captures what happened.
+    """
+    seq: int
+    action_type: str
+    outcome: str                    # "verified" | "failed" | "retry" | "skipped"
+    urn: str                        # CKP Action URN
+    kernel: str                     # CKP Kernel URN of who performed this
+    via_edge: str | None = None     # "COMPOSES" | "EXTENDS" | "REQUIRES" | None
+    intent: dict | None = None      # what the action wanted to achieve
+    evidence: dict | None = None    # what actually happened
+    before_state: dict | None = None
+    after_state: dict | None = None
+    diff_summary: str | None = None
+    attempt: int = 1
+    generated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    note: str = ""
+
+
+# ---------------------------------------------------------------------------
+# Session
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Session:
+    """A provenance session — one per ProvChain lifecycle.
+
+    session_id is a short UUID prefix for human-friendly reference.
+    All timestamps are UTC.
+    """
+    session_id: str
+    urn: str
+    label: str
+    kernel_action: str              # which action initiated this session
+    started_at: datetime
+    ended_at: datetime | None = None
+    was_associated_with: str = ""    # kernel URN
+    duration_ms: int | None = None
+    outcome: str = "in_progress"     # "complete" | "failed" | "partial" | "in_progress"
+    kernels_involved: list = field(default_factory=list)
+    actions: list = field(default_factory=list)  # list[ActionRecord]
+
+
+# ---------------------------------------------------------------------------
+# ProvChain — the session recorder
+# ---------------------------------------------------------------------------
+
+class ProvChain:
+    """Generic PROV-O session recorder for any Concept Kernel.
+
+    Creates a session file in storage/sessions/ and atomically overwrites
+    it on every recorded action. Automatically enforces retention policy.
+
+    Usage:
+        from cklib.prov import ProvChain
+
+        prov = ProvChain(
+            storage_dir="concepts/MyKernel",
+            label="my-operation",
+            kernel_urn="ckp://Kernel#MyKernel:v1.0",
+        )
+        prov.record_action(
+            action_type="process.data",
+            outcome="verified",
+            kernel="MyKernel",
+            intent={"target": "file.txt"},
+            evidence={"processed": True},
+        )
+        prov.close()
+    """
+
+    def __init__(
+        self,
+        storage_dir: str,
+        label: str,
+        kernel_urn: str,
+        kernel_action: str = "",
+        retention_hours: int = 48,
+    ):
+        self.storage_dir = storage_dir
+        self.label = label
+        self.kernel_urn = kernel_urn
+        self.kernel_action = kernel_action
+        self.retention_hours = retention_hours
+
+        # Derive kernel short-name from URN for involved-set
+        self._kernel_name = self._name_from_urn(kernel_urn)
+        self._kernels_involved: set = {self._kernel_name}
+
+        self.session_id = str(uuid.uuid4())[:8]
+        self._started_at = datetime.now(timezone.utc)
+        self._seq = 0
+        self._actions: list[ActionRecord] = []
+
+        # Session dir: storage/sessions/
+        self._sessions_dir = os.path.join(storage_dir, "storage", "sessions")
+        os.makedirs(self._sessions_dir, exist_ok=True)
+        self._file = os.path.join(self._sessions_dir, "%s.json" % self.session_id)
+
+        self._enforce_retention()
+
+        # Initial flush — creates the session file immediately
+        self._flush()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def record_action(
+        self,
+        action_type: str,
+        outcome: str,
+        kernel: str = "",
+        via_edge: str | None = None,
+        intent: dict | None = None,
+        evidence: dict | None = None,
+        before_state: dict | None = None,
+        after_state: dict | None = None,
+        attempt: int = 1,
+        note: str = "",
+    ) -> ActionRecord:
+        """Record an action in this session.
+
+        Args:
+            action_type: dotted action name (e.g. "task.create", "deploy.apply")
+            outcome:     "verified" | "failed" | "retry" | "skipped"
+            kernel:      short kernel name (e.g. "CK.Task") — URN built automatically
+            via_edge:    edge predicate if action crosses a kernel boundary
+            intent:      dict describing what the action wanted
+            evidence:    dict describing what actually happened
+            before_state: optional state snapshot before the action
+            after_state:  optional state snapshot after the action
+            attempt:     attempt number (1-based, for retries)
+            note:        human-readable note
+
+        Returns:
+            The ActionRecord that was created.
+        """
+        self._seq += 1
+
+        # Resolve kernel name and URN
+        if not kernel:
+            kernel = self._kernel_name
+        self._kernels_involved.add(kernel)
+
+        ts_ms = int(_time.time() * 1000)
+        action_urn = build_action_urn(kernel, action_type, ts_ms)
+
+        # Build kernel URN if not already one
+        if kernel.startswith("ckp://"):
+            kernel_urn = kernel
+        else:
+            kernel_urn = build_kernel_urn(kernel)
+
+        # Compute diff summary if both states provided
+        diff_summary = None
+        if before_state and after_state:
+            diff_summary = _compute_state_diff(before_state, after_state)
+
+        record = ActionRecord(
+            seq=self._seq,
+            action_type=action_type,
+            outcome=outcome,
+            urn=action_urn,
+            kernel=kernel_urn,
+            via_edge=via_edge,
+            intent=intent,
+            evidence=evidence,
+            before_state=before_state,
+            after_state=after_state,
+            diff_summary=diff_summary,
+            attempt=attempt,
+            generated_at=datetime.now(timezone.utc),
+            note=note,
+        )
+        self._actions.append(record)
+        self._flush()
+        self._emit_live(record)
+        return record
+
+    def close(self, outcome: str = "") -> Session:
+        """Finalize the session.
+
+        If outcome is not specified, it is derived from action results:
+        - all verified  -> "complete"
+        - any failed    -> "partial" (if also some verified) or "failed"
+        - no actions    -> "complete"
+        """
+        if not outcome:
+            outcome = self._derive_outcome()
+        session = self._flush(
+            ended_at=datetime.now(timezone.utc),
+            outcome=outcome,
+        )
+        return session
+
+    @property
+    def file_path(self) -> str:
+        """Path to the session JSON file."""
+        return self._file
+
+    @property
+    def action_count(self) -> int:
+        """Number of actions recorded so far."""
+        return len(self._actions)
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _flush(self, ended_at=None, outcome=None) -> Session:
+        """Atomic write of the full session to disk."""
+        duration_ms = None
+        if ended_at:
+            duration_ms = int((ended_at - self._started_at).total_seconds() * 1000)
+
+        session = Session(
+            session_id=self.session_id,
+            urn=build_action_urn(
+                self._kernel_name,
+                "session",
+                int(self._started_at.timestamp() * 1000),
+            ),
+            label=self.label,
+            kernel_action=self.kernel_action,
+            started_at=self._started_at,
+            ended_at=ended_at,
+            was_associated_with=self.kernel_urn,
+            duration_ms=duration_ms,
+            outcome=outcome or "in_progress",
+            kernels_involved=sorted(self._kernels_involved),
+            actions=self._actions,
+        )
+
+        data = _to_dict(session)
+        content = json.dumps(data, indent=2, default=_json_default)
+
+        # Atomic write via temp file + rename
+        tmp = self._file + ".tmp"
+        with open(tmp, "w") as f:
+            f.write(content)
+        os.replace(tmp, self._file)
+
+        return session
+
+    def _derive_outcome(self) -> str:
+        """Derive session outcome from recorded actions."""
+        if not self._actions:
+            return "complete"
+        n_failed = sum(1 for a in self._actions if a.outcome == "failed")
+        n_verified = sum(1 for a in self._actions if a.outcome == "verified")
+        if n_failed > 0 and n_verified == 0:
+            return "failed"
+        if n_failed > 0:
+            return "partial"
+        return "complete"
+
+    def _emit_live(self, action: ActionRecord):
+        """Emit a live provenance line to stderr for observability."""
+        ts = datetime.now(timezone.utc).strftime("%H:%M:%S.%f")[:12]
+
+        kernel_name = action.kernel or self._kernel_name
+        if "Kernel#" in kernel_name:
+            kernel_name = kernel_name.split("Kernel#")[1].split(":")[0]
+
+        sym = {
+            "verified": "+",
+            "failed": "x",
+            "retry": "~",
+            "skipped": "-",
+        }.get(action.outcome, "?")
+
+        detail = action.diff_summary or action.note or ""
+
+        attempt_str = ""
+        if action.attempt > 1:
+            attempt_str = " (attempt %d)" % action.attempt
+
+        print(
+            "[%s] %-24s %-24s %s %-8s %s%s" % (
+                ts, kernel_name, action.action_type,
+                sym, action.outcome, detail, attempt_str,
+            ),
+            file=sys.stderr,
+        )
+
+    def _enforce_retention(self):
+        """Delete session files older than retention_hours."""
+        cutoff = datetime.now(timezone.utc) - timedelta(hours=self.retention_hours)
+        for f in glob.glob(os.path.join(self._sessions_dir, "*.json")):
+            try:
+                mtime_ts = os.path.getmtime(f)
+                mtime = datetime.fromtimestamp(mtime_ts, tz=timezone.utc)
+                if mtime < cutoff:
+                    os.remove(f)
+            except OSError:
+                pass
+
+    @staticmethod
+    def _name_from_urn(urn: str) -> str:
+        """Extract kernel short name from a CKP Kernel URN."""
+        # ckp://Kernel#MyKernel:v1.0 -> MyKernel
+        if "Kernel#" in urn:
+            name = urn.split("Kernel#")[1]
+            if ":" in name:
+                name = name.split(":")[0]
+            return name
+        return urn
+
+
+# ---------------------------------------------------------------------------
+# Standalone helpers
+# ---------------------------------------------------------------------------
+
+def _compute_state_diff(before: dict, after: dict) -> str:
+    """Compute a human-readable diff summary between two state dicts.
+
+    Compares top-level keys. For nested dicts, reports changed keys.
+    Returns a pipe-separated summary string.
+    """
+    parts = []
+    all_keys = set(list(before.keys()) + list(after.keys()))
+    for key in sorted(all_keys):
+        bval = before.get(key)
+        aval = after.get(key)
+        if bval != aval:
+            if isinstance(bval, (int, float, str, bool)) or isinstance(aval, (int, float, str, bool)):
+                parts.append("%s: %s -> %s" % (key, bval, aval))
+            else:
+                parts.append("%s: changed" % key)
+    return " | ".join(parts) if parts else "no change"
+
+
+def list_sessions(storage_dir: str) -> list[dict]:
+    """Load all sessions from storage/sessions/, newest first.
+
+    Args:
+        storage_dir: the CK root directory (e.g. concepts/MyKernel/)
+
+    Returns:
+        List of session dicts, sorted by started_at descending.
+    """
+    sessions_dir = os.path.join(storage_dir, "storage", "sessions")
+    if not os.path.isdir(sessions_dir):
+        return []
+
+    sessions = []
+    for fpath in glob.glob(os.path.join(sessions_dir, "*.json")):
+        if fpath.endswith(".tmp"):
+            continue
+        try:
+            with open(fpath) as f:
+                data = json.load(f)
+            sessions.append(data)
+        except Exception:
+            continue
+
+    sessions.sort(
+        key=lambda s: s.get("started_at", ""),
+        reverse=True,
+    )
+    return sessions
+
+
+def get_session(storage_dir: str, session_id: str) -> dict | None:
+    """Load a session by ID or prefix match.
+
+    Args:
+        storage_dir: the CK root directory
+        session_id: full or prefix of session_id
+
+    Returns:
+        Session dict or None.
+    """
+    sessions_dir = os.path.join(storage_dir, "storage", "sessions")
+    if not os.path.isdir(sessions_dir):
+        return None
+
+    # Exact match
+    exact = os.path.join(sessions_dir, "%s.json" % session_id)
+    if os.path.exists(exact):
+        with open(exact) as f:
+            return json.load(f)
+
+    # Prefix match
+    for fpath in glob.glob(os.path.join(sessions_dir, "%s*.json" % session_id)):
+        if fpath.endswith(".tmp"):
+            continue
+        try:
+            with open(fpath) as f:
+                return json.load(f)
+        except Exception:
+            continue
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Verified action helper
+# ---------------------------------------------------------------------------
+
+def verified_action(
+    action_fn: Callable,
+    verify_fn: Callable[[], bool],
+    prov: ProvChain,
+    action_type: str,
+    kernel: str,
+    via_edge: str | None = None,
+    intent: dict | None = None,
+    max_attempts: int = 3,
+    settle_delay: float = 0.5,
+    capture_state_fn: Callable | None = None,
+) -> bool:
+    """Execute an action, verify it worked, record provenance.
+
+    Pattern: execute -> wait -> verify -> record outcome -> retry if needed.
+
+    This is the GENERIC version of YB.CLI's verified_focus_space /
+    verified_move_window / verified_bar_binding pattern. The caller provides:
+
+    - action_fn:        callable that performs the action (no return needed)
+    - verify_fn:        callable returning True if the action succeeded
+    - capture_state_fn: optional callable returning a dict snapshot of state
+
+    On success, records outcome="verified". On failure after all attempts,
+    records outcome="failed". Returns True/False.
+
+    Args:
+        action_fn:        the action to execute
+        verify_fn:        returns True if the action succeeded
+        prov:             ProvChain session to record into
+        action_type:      dotted action name
+        kernel:           kernel performing the action
+        via_edge:         edge predicate if crossing kernel boundary
+        intent:           dict describing what the action wants
+        max_attempts:     max retry count (default 3)
+        settle_delay:     seconds to wait after action before verify (default 0.5)
+        capture_state_fn: optional callable returning state dict
+
+    Returns:
+        True if verified within max_attempts, False otherwise.
+    """
+    import time
+
+    for attempt in range(1, max_attempts + 1):
+        # Capture before state
+        before_state = None
+        if capture_state_fn:
+            try:
+                before_state = capture_state_fn()
+            except Exception:
+                before_state = None
+
+        # Execute
+        try:
+            action_fn()
+        except Exception as e:
+            prov.record_action(
+                action_type=action_type,
+                outcome="failed",
+                kernel=kernel,
+                via_edge=via_edge,
+                intent=intent,
+                evidence={"error": str(e), "exception_type": type(e).__name__},
+                before_state=before_state,
+                attempt=attempt,
+                note="action raised: %s" % str(e),
+            )
+            if attempt < max_attempts:
+                time.sleep(settle_delay)
+                continue
+            return False
+
+        # Settle
+        if settle_delay > 0:
+            time.sleep(settle_delay)
+
+        # Capture after state
+        after_state = None
+        if capture_state_fn:
+            try:
+                after_state = capture_state_fn()
+            except Exception:
+                after_state = None
+
+        # Verify
+        try:
+            verified = verify_fn()
+        except Exception as e:
+            prov.record_action(
+                action_type=action_type,
+                outcome="failed",
+                kernel=kernel,
+                via_edge=via_edge,
+                intent=intent,
+                evidence={"verify_error": str(e)},
+                before_state=before_state,
+                after_state=after_state,
+                attempt=attempt,
+                note="verify raised: %s" % str(e),
+            )
+            if attempt < max_attempts:
+                time.sleep(settle_delay)
+                continue
+            return False
+
+        if verified:
+            prov.record_action(
+                action_type=action_type,
+                outcome="verified",
+                kernel=kernel,
+                via_edge=via_edge,
+                intent=intent,
+                evidence={"verified": True},
+                before_state=before_state,
+                after_state=after_state,
+                attempt=attempt,
+            )
+            return True
+        else:
+            prov.record_action(
+                action_type=action_type,
+                outcome="failed" if attempt == max_attempts else "retry",
+                kernel=kernel,
+                via_edge=via_edge,
+                intent=intent,
+                evidence={"verified": False},
+                before_state=before_state,
+                after_state=after_state,
+                attempt=attempt,
+                note="verification failed" if attempt == max_attempts else "retrying",
+            )
+            if attempt < max_attempts:
+                time.sleep(settle_delay)
+
+    return False