clm-kernel 0.0.0__py3-none-any.whl

clm/layer2/lineage.py

@@ -0,0 +1,322 @@
+"""Merkle Lineage Graph Engine (Sprint 395 / DEL-395-03).
+
+Tracks causal ancestry and Merkle provenance across multi-step Action workflows:
+    (Parent MCards, Action Hash, VCard Receipt) ──> Output MCard
+
+Guarantees:
+- AC-395-03-01: Every output MCard embeds immutable Merkle hashes of ancestor cards and actions.
+- AC-395-03-02: get_mcard_lineage() reconstructs complete provenance tree in <5ms.
+- AC-395-03-03: Visual export produces valid Mermaid flowchart and Graphviz DOT markdown.
+
+Layer 2 module (can import Layer 0, Layer 1, Layer 2).
+"""
+
+from __future__ import annotations
+
+import datetime
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+from clm.cross_cutting.domain_models import MCard
+
+
+@dataclass(frozen=True)
+class LineageNode:
+    """A node in the Merkle lineage graph."""
+
+    id: str
+    type: str  # "mcard", "action", "receipt"
+    label: str
+    hash: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class LineageEdge:
+    """A causal directed edge in the lineage graph."""
+
+    source: str
+    target: str
+    relation: str = "derives"  # "derives", "transformed_by", "certified_by"
+
+
+@dataclass
+class LineageDAG:
+    """Reconstructed directed acyclic graph of MCard provenance."""
+
+    target_uri: str
+    nodes: dict[str, LineageNode] = field(default_factory=dict)
+    edges: list[LineageEdge] = field(default_factory=list)
+    reconstruction_time_ms: float = 0.0
+
+    def add_node(self, node: LineageNode) -> None:
+        self.nodes[node.id] = node
+
+    def add_edge(self, edge: LineageEdge) -> None:
+        self.edges.append(edge)
+
+    def get_ancestors(self, uri: str) -> list[str]:
+        """Compute all ancestor URIs that contributed to the given node."""
+        parents_map: dict[str, list[str]] = {}
+        for e in self.edges:
+            parents_map.setdefault(e.target, []).append(e.source)
+
+        visited: set[str] = set()
+        queue = list(parents_map.get(uri, []))
+        while queue:
+            curr = queue.pop(0)
+            if curr not in visited:
+                visited.add(curr)
+                queue.extend(parents_map.get(curr, []))
+        return sorted(list(visited))
+
+    def get_root_inputs(self) -> list[str]:
+        """Return all root inputs (nodes with no incoming causal edges)."""
+        has_incoming = {e.target for e in self.edges}
+        return [nid for nid, node in self.nodes.items() if nid not in has_incoming and node.type == "mcard"]
+
+    def to_mermaid(self) -> str:
+        """Export lineage DAG to valid Mermaid flowchart markdown."""
+        lines = ["flowchart TD"]
+
+        # Define node styles
+        for nid, node in self.nodes.items():
+            safe_id = nid.replace(":", "_").replace("/", "_").replace(".", "_").replace("-", "_")
+            short_hash = node.hash[:8] if node.hash else ""
+            if node.type == "action":
+                label = f"⚡ Action: {node.label}\\n[{short_hash}]"
+                lines.append(f'    {safe_id}["{label}"]:::actionClass')
+            elif node.type == "receipt":
+                label = f"📜 Receipt: {node.label}\\n[{short_hash}]"
+                lines.append(f'    {safe_id}["{label}"]:::receiptClass')
+            else:
+                label = f"📦 {node.label}\\n[{short_hash}]"
+                lines.append(f'    {safe_id}["{label}"]:::mcardClass')
+
+        # Define edges
+        for e in self.edges:
+            src = e.source.replace(":", "_").replace("/", "_").replace(".", "_").replace("-", "_")
+            tgt = e.target.replace(":", "_").replace("/", "_").replace(".", "_").replace("-", "_")
+            if e.relation == "transformed_by":
+                lines.append(f"    {src} -->|transformed_by| {tgt}")
+            elif e.relation == "certified_by":
+                lines.append(f"    {tgt} -.->|certified_by| {src}")
+            else:
+                lines.append(f"    {src} -->|derives| {tgt}")
+
+        # Styles
+        lines.append("    classDef mcardClass fill:#e3f2fd,stroke:#1565c0,stroke-width:2px;")
+        lines.append("    classDef actionClass fill:#fff3e0,stroke:#e65100,stroke-width:2px;")
+        lines.append("    classDef receiptClass fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px;")
+
+        return "\n".join(lines)
+
+    def to_dot(self) -> str:
+        """Export lineage DAG to Graphviz DOT format."""
+        lines = ['digraph LineageDAG {', '    rankdir=TB;', '    node [shape=box, style=rounded, fontname="Helvetica"];']
+
+        for nid, node in self.nodes.items():
+            short_hash = node.hash[:8] if node.hash else ""
+            lines.append(f'    "{nid}" [label="{node.label}\\n({short_hash})"];')
+
+        for e in self.edges:
+            lines.append(f'    "{e.source}" -> "{e.target}" [label="{e.relation}"];')
+
+        lines.append("}")
+        return "\n".join(lines)
+
+
+@dataclass
+class LineageRecord:
+    """Immutable causal record of a single step derivation."""
+
+    output_uri: str
+    output_hash: str
+    parent_uris: list[str]
+    parent_hashes: list[str]
+    action_name: str
+    action_hash: str
+    receipt_uri: str | None
+    receipt_hash: str | None
+    step_id: str
+    timestamp: str = field(default_factory=lambda: datetime.datetime.now(datetime.timezone.utc).isoformat())
+
+
+class LineageRegistry:
+    """In-memory and persistent causal registry for MCard provenance."""
+
+    def __init__(self):
+        self._records: dict[str, LineageRecord] = {}  # output_uri -> record
+        self._hash_to_uri: dict[str, str] = {}        # output_hash -> uri
+        self._card_meta: dict[str, dict[str, Any]] = {}
+
+    def record_step_lineage(
+        self,
+        step_id: str,
+        action_name: str,
+        action_hash: str,
+        parent_cards: list[MCard] | dict[str, MCard],
+        output_cards: list[MCard] | dict[str, MCard],
+        receipt: MCard | None = None,
+    ) -> list[LineageRecord]:
+        """Record causal derivation edges for all outputs produced in a step."""
+        now = datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+        parents = list(parent_cards.values()) if isinstance(parent_cards, dict) else list(parent_cards)
+        outputs = list(output_cards.values()) if isinstance(output_cards, dict) else list(output_cards)
+
+        def _extract_hash(card: Any) -> str:
+            if card is None:
+                return ""
+            h = getattr(card, "hash", None) or getattr(card, "content_hash", None)
+            return str(h) if h else ""
+
+        parent_uris = [p.uri for p in parents if p and getattr(p, "uri", None)]
+        parent_hashes = [_extract_hash(p) for p in parents if p]
+
+        receipt_uri = receipt.uri if receipt and getattr(receipt, "uri", None) else None
+        receipt_hash = _extract_hash(receipt) if receipt else None
+
+        records: list[LineageRecord] = []
+        for out in outputs:
+            out_hash = _extract_hash(out)
+            out_uri = out.uri if getattr(out, "uri", None) else f"mcard://outputs/{out_hash[:16]}"
+
+            # Embed Merkle lineage into MCard metadata (AC-395-03-01)
+            lineage_block = {
+                "parents": parent_uris,
+                "parent_hashes": parent_hashes,
+                "action": {
+                    "name": action_name,
+                    "hash": action_hash,
+                },
+                "receipt": {
+                    "uri": receipt_uri,
+                    "hash": receipt_hash,
+                },
+                "step_id": step_id,
+                "timestamp": now,
+            }
+            if out.metadata is None:
+                out.metadata = {}
+            out.metadata["lineage"] = lineage_block
+
+            rec = LineageRecord(
+                output_uri=out_uri,
+                output_hash=out_hash,
+                parent_uris=parent_uris,
+                parent_hashes=parent_hashes,
+                action_name=action_name,
+                action_hash=action_hash,
+                receipt_uri=receipt_uri,
+                receipt_hash=receipt_hash,
+                step_id=step_id,
+                timestamp=now,
+            )
+            self._records[out_uri] = rec
+            norm_uri = self._normalize_uri(out_uri)
+            if norm_uri != out_uri:
+                self._records[norm_uri] = rec
+            if out_hash:
+                self._hash_to_uri[out_hash] = out_uri
+            records.append(rec)
+
+            # Record parents metadata
+            for p in parents:
+                if p.uri and p.uri not in self._card_meta:
+                    self._card_meta[p.uri] = {
+                        "label": p.uri.split("/")[-1],
+                        "hash": _extract_hash(p),
+                        "type": "mcard",
+                    }
+                    self._card_meta[self._normalize_uri(p.uri)] = self._card_meta[p.uri]
+            self._card_meta[out_uri] = {
+                "label": out_uri.split("/")[-1],
+                "hash": out_hash,
+                "type": "mcard",
+            }
+            self._card_meta[norm_uri] = self._card_meta[out_uri]
+
+        return records
+
+    @staticmethod
+    def _normalize_uri(uri: str) -> str:
+        if not uri:
+            return ""
+        if uri.startswith("mcard://"):
+            return f"mcard:{uri[8:].lstrip('/')}"
+        if uri.startswith("mcard:"):
+            return f"mcard://{uri[6:].lstrip('/')}"
+        return f"mcard:{uri.lstrip('/')}"
+
+    def get_mcard_lineage(self, mcard_uri_or_hash: str) -> LineageDAG:
+        """Reconstruct complete provenance tree in <5ms (AC-395-03-02)."""
+        t_start = time.perf_counter()
+
+        target_uri = self._hash_to_uri.get(mcard_uri_or_hash, mcard_uri_or_hash)
+        dag = LineageDAG(target_uri=target_uri)
+
+        visited: set[str] = set()
+        queue = [target_uri]
+
+        while queue:
+            curr_uri = queue.pop(0)
+            if curr_uri in visited:
+                continue
+            visited.add(curr_uri)
+
+            # Node for this card
+            meta = self._card_meta.get(curr_uri) or self._card_meta.get(self._normalize_uri(curr_uri), {})
+            node_label = meta.get("label", curr_uri.split("/")[-1])
+            node_hash = meta.get("hash", "")
+            dag.add_node(LineageNode(id=curr_uri, type="mcard", label=node_label, hash=node_hash))
+
+            # Find if this card has derivation records
+            rec = self._records.get(curr_uri) or self._records.get(self._normalize_uri(curr_uri))
+            if rec:
+                # Add Action node and edges
+                action_id = f"action:{rec.action_name}:{rec.step_id}"
+                dag.add_node(LineageNode(id=action_id, type="action", label=rec.action_name, hash=rec.action_hash))
+
+                # Edges: Parents -> Action
+                for p_uri in rec.parent_uris:
+                    dag.add_edge(LineageEdge(source=p_uri, target=action_id, relation="derives"))
+                    if p_uri not in visited:
+                        queue.append(p_uri)
+
+                # Edge: Action -> Output
+                dag.add_edge(LineageEdge(source=action_id, target=curr_uri, relation="transformed_by"))
+
+                # Receipt node and edge if present
+                if rec.receipt_uri:
+                    dag.add_node(LineageNode(
+                        id=rec.receipt_uri,
+                        type="receipt",
+                        label="VCardReceipt",
+                        hash=rec.receipt_hash or "",
+                    ))
+                    dag.add_edge(LineageEdge(source=rec.receipt_uri, target=curr_uri, relation="certified_by"))
+
+        t_elapsed = (time.perf_counter() - t_start) * 1000.0
+        dag.reconstruction_time_ms = round(t_elapsed, 3)
+        return dag
+
+    def reset(self) -> None:
+        """Reset the registry state."""
+        self._records.clear()
+        self._hash_to_uri.clear()
+        self._card_meta.clear()
+
+
+# Global default registry
+_default_lineage_registry = LineageRegistry()
+
+
+def get_default_lineage_registry() -> LineageRegistry:
+    return _default_lineage_registry
+
+
+def get_mcard_lineage(mcard_uri_or_hash: str) -> LineageDAG:
+    """Public helper to query MCard provenance using the default registry."""
+    return _default_lineage_registry.get_mcard_lineage(mcard_uri_or_hash)

clm/layer2/mcard_fs.py

@@ -0,0 +1,262 @@
+"""MCard Universal Virtual Filesystem Engine (Sprint 191/193/383).
+
+Python parity with Rust `clm_rust_core::mcard_fs::MCardFileSystem`.
+Relocated to Layer 2 (INV-00, INV-09, DEL-383-05).
+Enforces:
+1. Zero-FS Hermeticity (INV-09): Works with in-memory SQLite without host disk I/O.
+2. Handle Registry Singularity (INV-04) & Tri-Database Separation (INV-05):
+   Wraps canonical `card`, `handle_registry`, and `handle_history` tables.
+3. Content-Addressable Identity (INV-02): BLAKE3 verification on all reads.
+"""
+
+from __future__ import annotations
+
+import datetime
+import json
+import sqlite3
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+from clm.cross_cutting.domain_models import MCard
+from clm.cross_cutting.domain_types import AgentDid, ContentHash, make_content_hash
+
+
+class MCardStat(BaseModel):
+    """Metadata stat for a virtual MCard file."""
+
+    model_config = ConfigDict(frozen=True)
+
+    uri: str
+    hash: ContentHash
+    size_bytes: int
+    created_at: str
+    updated_at: str
+
+
+class MCardFileSystem:
+    """Universal immutable filesystem engine mapping URIs to content-addressed MCards."""
+
+    def __init__(self, conn: sqlite3.Connection | None = None, context_prefix: str = "") -> None:
+        self.conn = conn or sqlite3.connect(":memory:")
+        self.context_prefix = context_prefix.strip("/")
+        self._init_schema()
+
+    def with_prefix(self, prefix: str) -> MCardFileSystem:
+        """Returns a new MCardFileSystem handle isolated to the given context prefix."""
+        return MCardFileSystem(conn=self.conn, context_prefix=prefix)
+
+    def canonicalize_uri(self, path: str) -> str:
+        """Resolves canonical URI with context prefix scoping (DOD-191-04)."""
+        clean = path.lstrip("/")
+        if clean.startswith("mcard:"):
+            rest_clean = clean[len("mcard:"):].lstrip("/")
+            if not self.context_prefix or rest_clean.startswith(self.context_prefix):
+                return f"mcard:{rest_clean}"
+            else:
+                return f"mcard:{self.context_prefix}/{rest_clean}"
+        elif not self.context_prefix:
+            return f"mcard:{clean}"
+        else:
+            return f"mcard:{self.context_prefix}/{clean}"
+
+    def _init_schema(self) -> None:
+        with self.conn:
+            self.conn.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS card (
+                    hash TEXT PRIMARY KEY,
+                    content TEXT NOT NULL,
+                    g_time TEXT
+                );
+
+                CREATE TABLE IF NOT EXISTS handle_registry (
+                    handle TEXT PRIMARY KEY,
+                    current_hash TEXT NOT NULL,
+                    created_at TEXT NOT NULL,
+                    updated_at TEXT NOT NULL
+                );
+
+                CREATE TABLE IF NOT EXISTS handle_history (
+                    handle TEXT NOT NULL,
+                    previous_hash TEXT NOT NULL,
+                    changed_at TEXT NOT NULL
+                );
+
+                CREATE INDEX IF NOT EXISTS idx_handle_reg_prefix ON handle_registry(handle);
+                """
+            )
+
+    def write_file(
+        self,
+        uri: str,
+        payload: Any,
+        author: AgentDid | str,
+        metadata: dict[str, Any] | None = None,
+    ) -> MCard:
+        """Writes an MCard into the filesystem and updates the handle registry."""
+        canonical_uri = self.canonicalize_uri(uri)
+        now = datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+        def _do_write() -> MCard:
+            # 1. Determine existing handle state
+            cur = self.conn.execute(
+                "SELECT current_hash, created_at FROM handle_registry WHERE handle = ?",
+                (canonical_uri,),
+            )
+            existing = cur.fetchone()
+
+            if existing:
+                prev_hash, created_at = existing
+                hist_cur = self.conn.execute(
+                    "SELECT count(*) FROM handle_history WHERE handle = ?", (canonical_uri,)
+                )
+                count = hist_cur.fetchone()[0]
+                seq = count + 2
+            else:
+                prev_hash = None
+                created_at = now
+                seq = 1
+
+            # 2. Construct canonical MCard
+            mcard = MCard.create(
+                uri=canonical_uri,
+                payload=payload,
+                author=author,
+                sequence=seq,
+                metadata=metadata,
+            )
+            serialized = json.dumps(mcard.model_dump(), sort_keys=True)
+            hash_str = str(mcard.hash)
+
+            # 3. Insert CAS card blob
+            self.conn.execute(
+                "INSERT OR IGNORE INTO card (hash, content, g_time) VALUES (?, ?, ?)",
+                (hash_str, serialized, now),
+            )
+
+            # 4. Update history if hash changed
+            if prev_hash and prev_hash != hash_str:
+                self.conn.execute(
+                    "INSERT INTO handle_history (handle, previous_hash, changed_at) VALUES (?, ?, ?)",
+                    (canonical_uri, prev_hash, now),
+                )
+
+            # 5. Insert or replace handle entry
+            self.conn.execute(
+                "INSERT OR REPLACE INTO handle_registry (handle, current_hash, created_at, updated_at) "
+                "VALUES (?, ?, ?, ?)",
+                (canonical_uri, hash_str, created_at, now),
+            )
+
+            return mcard
+
+        if self.conn.in_transaction:
+            return _do_write()
+        else:
+            with self.conn:
+                return _do_write()
+
+    def read_file(self, uri: str) -> MCard:
+        """Reads and cryptographically verifies an MCard by handle URI."""
+        canonical_uri = self.canonicalize_uri(uri)
+        cur = self.conn.execute(
+            "SELECT current_hash FROM handle_registry WHERE handle = ?", (canonical_uri,)
+        )
+        row = cur.fetchone()
+        if not row:
+            raise KeyError(f"Handle '{canonical_uri}' not found in MCardFs")
+
+        current_hash = row[0]
+        card_cur = self.conn.execute(
+            "SELECT content FROM card WHERE hash = ?", (current_hash,)
+        )
+        card_row = card_cur.fetchone()
+        if not card_row:
+            raise ValueError(f"Card content for hash '{current_hash}' corrupted or missing")
+
+        data = json.loads(card_row[0])
+        mcard = MCard(**data)
+
+        # Cryptographic integrity check (INV-02)
+        from clm.layer0.utils import clm_hash_string
+        canonical_serialized = json.dumps(mcard.payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        expected_hash = clm_hash_string(canonical_serialized)
+
+        if expected_hash != current_hash:
+            raise ValueError(
+                f"Cryptographic hash mismatch for '{canonical_uri}': registry={current_hash} computed={expected_hash}"
+            )
+
+        return mcard
+
+    def exists(self, uri: str) -> bool:
+        """Checks if a handle exists in the filesystem."""
+        canonical_uri = self.canonicalize_uri(uri)
+        cur = self.conn.execute(
+            "SELECT count(*) FROM handle_registry WHERE handle = ?", (canonical_uri,)
+        )
+        return cur.fetchone()[0] > 0
+
+    def stat_file(self, uri: str) -> MCardStat:
+        """Returns stat metadata for a handle URI."""
+        canonical_uri = self.canonicalize_uri(uri)
+        cur = self.conn.execute(
+            "SELECT current_hash, created_at, updated_at FROM handle_registry WHERE handle = ?",
+            (canonical_uri,),
+        )
+        row = cur.fetchone()
+        if not row:
+            raise KeyError(f"Handle '{canonical_uri}' not found")
+
+        current_hash, created_at, updated_at = row
+        card_cur = self.conn.execute(
+            "SELECT length(content) FROM card WHERE hash = ?", (current_hash,)
+        )
+        card_row = card_cur.fetchone()
+        size_bytes = card_row[0] if card_row else 0
+
+        return MCardStat(
+            uri=canonical_uri,
+            hash=make_content_hash(current_hash),
+            size_bytes=size_bytes,
+            created_at=created_at,
+            updated_at=updated_at,
+        )
+
+    def list_dir(self, prefix: str) -> list[str]:
+        """Lists all handles matching the specified prefix."""
+        canonical_prefix = self.canonicalize_uri(prefix)
+        if canonical_prefix.endswith("/"):
+            pattern = f"{canonical_prefix}%"
+        elif canonical_prefix == "mcard:":
+            pattern = "mcard:%"
+        else:
+            pattern = f"{canonical_prefix}/%"
+
+        cur = self.conn.execute(
+            "SELECT handle FROM handle_registry WHERE handle LIKE ? ORDER BY handle ASC",
+            (pattern,),
+        )
+        return [r[0] for r in cur.fetchall()]
+
+    def delete_file(self, uri: str) -> None:
+        """Deregisters a handle URI while preserving the CAS content blob."""
+        canonical_uri = self.canonicalize_uri(uri)
+        now = datetime.datetime.now(datetime.timezone.utc).isoformat()
+        with self.conn:
+            cur = self.conn.execute(
+                "SELECT current_hash FROM handle_registry WHERE handle = ?", (canonical_uri,)
+            )
+            row = cur.fetchone()
+            if not row:
+                raise KeyError(f"Handle '{canonical_uri}' not found for deletion")
+
+            self.conn.execute(
+                "INSERT INTO handle_history (handle, previous_hash, changed_at) VALUES (?, ?, ?)",
+                (canonical_uri, row[0], now),
+            )
+            self.conn.execute("DELETE FROM handle_registry WHERE handle = ?", (canonical_uri,))
+
+
+__all__ = ["MCardStat", "MCardFileSystem"]

clm/layer2/merkle.py

@@ -0,0 +1,11 @@
+"""Layer 2: Merkle Tree & Directory Ingestion.
+
+Canonical Path: kernel/scripts/clm/layer2/merkle.py
+Parity: Rust `src/layer2/merkle.rs`, TypeScript `src/layer2/merkleTree.ts`
+"""
+
+from __future__ import annotations
+
+from clm.layer0.algebra_ingest import directory_fs_scan_py, directory_to_db_py
+
+__all__ = ["directory_fs_scan_py", "directory_to_db_py"]