icdev 1.0.0__py3-none-any.whl

icdev/tools/observability/provenance/prov_recorder.py

@@ -0,0 +1,324 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""PROV-AGENT Provenance Recorder — W3C PROV standard (D287).
+
+Records Entity/Activity/Relation provenance data in SQLite.
+Auto-populates from span completion callbacks when tracing is active.
+
+Entity types: prompt, response, document, code, report, artifact
+Activity types: tool_invocation, llm_call, decision, review, generation
+Relation types: wasGeneratedBy, used, wasInformedBy, wasDerivedFrom, wasAttributedTo
+
+Usage:
+    from icdev.tools.observability.provenance.prov_recorder import ProvRecorder
+from icdev._paths import get_project_root
+    recorder = ProvRecorder()
+
+    entity_id = recorder.record_entity("prompt", "User query", content_hash="abc...")
+    activity_id = recorder.record_activity("llm_call", "Claude Opus invocation")
+    recorder.record_relation("wasGeneratedBy", entity_id, activity_id)
+"""
+
+import hashlib
+import json
+import logging
+import os
+import sqlite3
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger("icdev.observability.provenance")
+
+BASE_DIR = get_project_root()
+DB_PATH = BASE_DIR / "data" / "icdev.db"
+
+
+class ProvRecorder:
+    """W3C PROV-AGENT provenance recorder (D287).
+
+    Records entities, activities, and relations in append-only SQLite tables.
+    Follows NIST AI RMF MEASURE 2.5 (provenance).
+    """
+
+    def __init__(
+        self,
+        db_path: Optional[Path] = None,
+        agent_id: Optional[str] = None,
+        project_id: Optional[str] = None,
+        classification: str = "CUI",
+    ):
+        self._db_path = db_path or DB_PATH
+        self._agent_id = agent_id
+        self._project_id = project_id
+        self._classification = classification
+
+    def _content_tracing_enabled(self) -> bool:
+        return os.environ.get("ICDEV_CONTENT_TRACING_ENABLED", "").lower() in ("true", "1", "yes")
+
+    def record_entity(
+        self,
+        entity_type: str,
+        label: str,
+        content: Optional[str] = None,
+        content_hash: Optional[str] = None,
+        attributes: Optional[Dict[str, Any]] = None,
+        trace_id: Optional[str] = None,
+        span_id: Optional[str] = None,
+    ) -> Optional[str]:
+        """Record a provenance entity (D287).
+
+        Args:
+            entity_type: prompt, response, document, code, report, artifact.
+            label: Human-readable label.
+            content: Entity content (only stored if content tracing enabled).
+            content_hash: SHA-256 hash (computed from content if not provided).
+            attributes: Additional metadata.
+            trace_id: Associated trace ID.
+            span_id: Associated span ID.
+
+        Returns:
+            Entity ID or None if DB unavailable.
+        """
+        entity_id = str(uuid.uuid4())
+
+        if content and not content_hash:
+            content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+        stored_content = content if self._content_tracing_enabled() else None
+
+        if not self._db_path.exists():
+            return None
+
+        try:
+            conn = sqlite3.connect(str(self._db_path))
+            conn.execute(
+                """INSERT INTO prov_entities
+                   (id, entity_type, label, content_hash, content, attributes,
+                    trace_id, span_id, agent_id, project_id, classification)
+                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                (
+                    entity_id, entity_type, label, content_hash, stored_content,
+                    json.dumps(attributes or {}),
+                    trace_id, span_id,
+                    self._agent_id, self._project_id, self._classification,
+                ),
+            )
+            conn.commit()
+            conn.close()
+            return entity_id
+        except sqlite3.Error as e:
+            logger.error("Failed to record entity: %s", e)
+            return None
+
+    def record_activity(
+        self,
+        activity_type: str,
+        label: str,
+        start_time: Optional[str] = None,
+        end_time: Optional[str] = None,
+        attributes: Optional[Dict[str, Any]] = None,
+        trace_id: Optional[str] = None,
+        span_id: Optional[str] = None,
+    ) -> Optional[str]:
+        """Record a provenance activity (D287).
+
+        Args:
+            activity_type: tool_invocation, llm_call, decision, review, generation.
+            label: Human-readable label.
+            start_time: ISO format start time.
+            end_time: ISO format end time.
+            attributes: Additional metadata.
+            trace_id: Associated trace ID.
+            span_id: Associated span ID.
+
+        Returns:
+            Activity ID or None.
+        """
+        activity_id = str(uuid.uuid4())
+        now = datetime.now(timezone.utc).isoformat()
+
+        if not self._db_path.exists():
+            return None
+
+        try:
+            conn = sqlite3.connect(str(self._db_path))
+            conn.execute(
+                """INSERT INTO prov_activities
+                   (id, activity_type, label, start_time, end_time, attributes,
+                    trace_id, span_id, agent_id, project_id, classification)
+                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                (
+                    activity_id, activity_type, label,
+                    start_time or now, end_time,
+                    json.dumps(attributes or {}),
+                    trace_id, span_id,
+                    self._agent_id, self._project_id, self._classification,
+                ),
+            )
+            conn.commit()
+            conn.close()
+            return activity_id
+        except sqlite3.Error as e:
+            logger.error("Failed to record activity: %s", e)
+            return None
+
+    def record_relation(
+        self,
+        relation_type: str,
+        subject_id: str,
+        object_id: str,
+        attributes: Optional[Dict[str, Any]] = None,
+        trace_id: Optional[str] = None,
+    ) -> bool:
+        """Record a provenance relation (D287).
+
+        Args:
+            relation_type: wasGeneratedBy, used, wasInformedBy, wasDerivedFrom, wasAttributedTo.
+            subject_id: Source entity/activity ID.
+            object_id: Target entity/activity ID.
+            attributes: Additional metadata.
+            trace_id: Associated trace ID.
+
+        Returns:
+            True if recorded, False on failure.
+        """
+        if not self._db_path.exists():
+            return False
+
+        try:
+            conn = sqlite3.connect(str(self._db_path))
+            conn.execute(
+                """INSERT INTO prov_relations
+                   (relation_type, subject_id, object_id, attributes,
+                    trace_id, project_id, classification)
+                   VALUES (?, ?, ?, ?, ?, ?, ?)""",
+                (
+                    relation_type, subject_id, object_id,
+                    json.dumps(attributes or {}),
+                    trace_id, self._project_id, self._classification,
+                ),
+            )
+            conn.commit()
+            conn.close()
+            return True
+        except sqlite3.Error as e:
+            logger.error("Failed to record relation: %s", e)
+            return False
+
+    def get_lineage(
+        self,
+        entity_id: str,
+        direction: str = "backward",
+        max_depth: int = 50,
+    ) -> List[Dict]:
+        """Query provenance lineage for an entity.
+
+        Args:
+            entity_id: Starting entity ID.
+            direction: "backward" (what produced this?) or "forward" (what did this produce?).
+            max_depth: Maximum traversal depth.
+
+        Returns:
+            List of relation dicts forming the lineage chain.
+        """
+        if not self._db_path.exists():
+            return []
+
+        results = []
+        visited = set()
+        queue = [entity_id]
+
+        try:
+            conn = sqlite3.connect(str(self._db_path))
+            conn.row_factory = sqlite3.Row
+
+            depth = 0
+            while queue and depth < max_depth:
+                current = queue.pop(0)
+                if current in visited:
+                    continue
+                visited.add(current)
+
+                if direction == "backward":
+                    rows = conn.execute(
+                        "SELECT * FROM prov_relations WHERE subject_id = ?",
+                        (current,),
+                    ).fetchall()
+                    for row in rows:
+                        results.append(dict(row))
+                        queue.append(row["object_id"])
+                else:
+                    rows = conn.execute(
+                        "SELECT * FROM prov_relations WHERE object_id = ?",
+                        (current,),
+                    ).fetchall()
+                    for row in rows:
+                        results.append(dict(row))
+                        queue.append(row["subject_id"])
+
+                depth += 1
+
+            conn.close()
+        except sqlite3.Error as e:
+            logger.error("Lineage query error: %s", e)
+
+        return results
+
+    def export_prov_json(self, project_id: Optional[str] = None) -> Dict:
+        """Export provenance data in PROV-JSON format.
+
+        Args:
+            project_id: Filter by project. Defaults to recorder's project_id.
+
+        Returns:
+            PROV-JSON dictionary.
+        """
+        pid = project_id or self._project_id
+        if not self._db_path.exists():
+            return {"entity": {}, "activity": {}, "relation": []}
+
+        try:
+            conn = sqlite3.connect(str(self._db_path))
+            conn.row_factory = sqlite3.Row
+
+            where = "WHERE project_id = ?" if pid else ""
+            params = (pid,) if pid else ()
+
+            entities = conn.execute(f"SELECT * FROM prov_entities {where}", params).fetchall()
+            activities = conn.execute(f"SELECT * FROM prov_activities {where}", params).fetchall()
+            relations = conn.execute(f"SELECT * FROM prov_relations {where}", params).fetchall()
+            conn.close()
+
+            return {
+                "prefix": {"prov": "http://www.w3.org/ns/prov#", "icdev": "urn:icdev:"},
+                "entity": {
+                    e["id"]: {
+                        "prov:type": e["entity_type"],
+                        "prov:label": e["label"],
+                        "icdev:content_hash": e["content_hash"],
+                    }
+                    for e in entities
+                },
+                "activity": {
+                    a["id"]: {
+                        "prov:type": a["activity_type"],
+                        "prov:label": a["label"],
+                        "prov:startTime": a["start_time"],
+                        "prov:endTime": a["end_time"],
+                    }
+                    for a in activities
+                },
+                "relation": [
+                    {
+                        "prov:type": r["relation_type"],
+                        "prov:subject": r["subject_id"],
+                        "prov:object": r["object_id"],
+                    }
+                    for r in relations
+                ],
+            }
+        except sqlite3.Error as e:
+            logger.error("PROV-JSON export error: %s", e)
+            return {"entity": {}, "activity": {}, "relation": []}

icdev/tools/observability/shap/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""ICDEV AgentSHAP — Monte Carlo Shapley value tool attribution (D288)."""

icdev/tools/observability/shap/agent_shap.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""AgentSHAP — Monte Carlo Shapley value tool attribution (D288).
+
+Computes Shapley values for tool importance in LLM agent traces.
+Uses Monte Carlo sampling (stdlib `random`, D22 air-gap safe).
+
+Reference: arXiv:2512.12597 — 0.945 consistency, model-agnostic.
+
+Usage:
+    from icdev.tools.observability.shap.agent_shap import AgentSHAP
+from icdev._paths import get_project_root
+    shap = AgentSHAP()
+    results = shap.analyze_trace(trace_id="abc123", iterations=1000)
+
+CLI:
+    python tools/observability/shap/agent_shap.py --trace-id abc123 --json
+    python tools/observability/shap/agent_shap.py --project-id proj-123 --last-n 10 --json
+"""
+
+import argparse
+import json
+import logging
+import math
+import random
+import sqlite3
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+logger = logging.getLogger("icdev.observability.shap")
+
+BASE_DIR = get_project_root()
+DB_PATH = BASE_DIR / "data" / "icdev.db"
+
+
+class AgentSHAP:
+    """Monte Carlo Shapley value computation for tool attribution (D288).
+
+    Determines which tools contributed most to trace outcome using
+    coalition sampling and marginal contribution measurement.
+    """
+
+    def __init__(
+        self,
+        db_path: Optional[Path] = None,
+        seed: Optional[int] = None,
+    ):
+        self._db_path = db_path or DB_PATH
+        if seed is not None:
+            random.seed(seed)
+
+    def analyze_trace(
+        self,
+        trace_id: str,
+        iterations: int = 1000,
+        outcome_metric: str = "success",
+    ) -> Dict[str, Any]:
+        """Compute Shapley values for tools in a single trace.
+
+        Args:
+            trace_id: Trace ID to analyze.
+            iterations: Monte Carlo sampling iterations.
+            outcome_metric: Metric to evaluate ("success", "duration", "quality").
+
+        Returns:
+            Dict with tool attributions and metadata.
+        """
+        spans = self._get_trace_spans(trace_id)
+        if not spans:
+            return {"error": "No spans found for trace", "trace_id": trace_id}
+
+        # Extract unique tool names from spans
+        tools = self._extract_tools(spans)
+        if not tools:
+            return {"error": "No tool calls in trace", "trace_id": trace_id}
+
+        # Compute outcome for the full trace
+        full_outcome = self._evaluate_outcome(spans, tools, outcome_metric)
+
+        # Monte Carlo Shapley estimation
+        shapley_values: Dict[str, List[float]] = {t: [] for t in tools}
+
+        for _ in range(iterations):
+            # Random permutation of tools
+            perm = list(tools)
+            random.shuffle(perm)
+
+            coalition: Set[str] = set()
+            prev_value = self._evaluate_outcome(spans, coalition, outcome_metric)
+
+            for tool in perm:
+                coalition.add(tool)
+                curr_value = self._evaluate_outcome(spans, coalition, outcome_metric)
+                marginal = curr_value - prev_value
+                shapley_values[tool].append(marginal)
+                prev_value = curr_value
+
+        # Compute statistics
+        results = {}
+        for tool in tools:
+            values = shapley_values[tool]
+            mean = sum(values) / len(values)
+            variance = sum((v - mean) ** 2 for v in values) / len(values)
+            std = math.sqrt(variance)
+            # 95% confidence interval
+            z = 1.96
+            ci_low = mean - z * std / math.sqrt(len(values))
+            ci_high = mean + z * std / math.sqrt(len(values))
+
+            results[tool] = {
+                "shapley_value": round(mean, 6),
+                "confidence_low": round(ci_low, 6),
+                "confidence_high": round(ci_high, 6),
+                "std": round(std, 6),
+                "coalition_size": len(tools),
+            }
+
+        # Normalize to [0, 1]
+        total = sum(abs(r["shapley_value"]) for r in results.values())
+        if total > 0:
+            for tool in results:
+                results[tool]["normalized"] = round(
+                    abs(results[tool]["shapley_value"]) / total, 6
+                )
+        else:
+            for tool in results:
+                results[tool]["normalized"] = round(1.0 / len(tools), 6)
+
+        # Store results
+        self._store_attributions(trace_id, results, iterations, outcome_metric)
+
+        return {
+            "trace_id": trace_id,
+            "tool_count": len(tools),
+            "iterations": iterations,
+            "outcome_metric": outcome_metric,
+            "full_outcome": full_outcome,
+            "attributions": results,
+        }
+
+    def _extract_tools(self, spans: List[Dict]) -> List[str]:
+        """Extract unique tool names from MCP tool call spans."""
+        tools = set()
+        for span in spans:
+            if span.get("name") == "mcp.tool_call":
+                attrs = json.loads(span.get("attributes", "{}"))
+                tool_name = attrs.get("mcp.tool.name")
+                if tool_name:
+                    tools.add(tool_name)
+        return sorted(tools)
+
+    def _evaluate_outcome(
+        self,
+        spans: List[Dict],
+        coalition: Set[str],
+        metric: str,
+    ) -> float:
+        """Evaluate outcome for a coalition of tools.
+
+        The outcome function measures trace quality with only the
+        coalition tools active. For excluded tools, their spans are
+        treated as if they returned errors.
+        """
+        if not coalition:
+            return 0.0
+
+        relevant_spans = []
+        for span in spans:
+            if span.get("name") == "mcp.tool_call":
+                attrs = json.loads(span.get("attributes", "{}"))
+                tool_name = attrs.get("mcp.tool.name", "")
+                if tool_name in coalition:
+                    relevant_spans.append(span)
+
+        if not relevant_spans:
+            return 0.0
+
+        if metric == "success":
+            # Fraction of tool calls that succeeded
+            ok_count = sum(
+                1 for s in relevant_spans if s.get("status_code") == "OK"
+            )
+            return ok_count / len(relevant_spans)
+        elif metric == "duration":
+            # Inverse of total duration (lower is better, so invert)
+            total_ms = sum(s.get("duration_ms", 0) for s in relevant_spans)
+            return 1.0 / (1.0 + total_ms / 1000.0)
+        else:
+            # Default: success rate
+            ok_count = sum(
+                1 for s in relevant_spans if s.get("status_code") == "OK"
+            )
+            return ok_count / len(relevant_spans)
+
+    def _get_trace_spans(self, trace_id: str) -> List[Dict]:
+        """Fetch all spans for a trace."""
+        if not self._db_path.exists():
+            return []
+
+        try:
+            conn = sqlite3.connect(str(self._db_path))
+            conn.row_factory = sqlite3.Row
+            rows = conn.execute(
+                "SELECT * FROM otel_spans WHERE trace_id = ? ORDER BY start_time",
+                (trace_id,),
+            ).fetchall()
+            conn.close()
+            return [dict(row) for row in rows]
+        except sqlite3.Error:
+            return []
+
+    def _store_attributions(
+        self,
+        trace_id: str,
+        results: Dict[str, Dict],
+        iterations: int,
+        outcome_metric: str,
+    ) -> None:
+        """Store Shapley attributions in DB (append-only, D6)."""
+        if not self._db_path.exists():
+            return
+
+        try:
+            conn = sqlite3.connect(str(self._db_path))
+            for tool_name, attrs in results.items():
+                conn.execute(
+                    """INSERT INTO shap_attributions
+                       (trace_id, tool_name, shapley_value, coalition_size,
+                        confidence_low, confidence_high, outcome_metric,
+                        outcome_value, analysis_params)
+                       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                    (
+                        trace_id,
+                        tool_name,
+                        attrs["shapley_value"],
+                        attrs["coalition_size"],
+                        attrs["confidence_low"],
+                        attrs["confidence_high"],
+                        outcome_metric,
+                        attrs.get("normalized", 0),
+                        json.dumps({"iterations": iterations}),
+                    ),
+                )
+            conn.commit()
+            conn.close()
+        except sqlite3.Error as e:
+            logger.error("Failed to store SHAP attributions: %s", e)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="AgentSHAP Tool Attribution (D288)")
+    parser.add_argument("--trace-id", help="Trace ID to analyze")
+    parser.add_argument("--project-id", help="Project ID for batch analysis")
+    parser.add_argument("--last-n", type=int, default=5, help="Last N traces to analyze")
+    parser.add_argument("--iterations", type=int, default=1000, help="MC iterations")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    args = parser.parse_args()
+
+    shap = AgentSHAP()
+
+    if args.trace_id:
+        result = shap.analyze_trace(args.trace_id, iterations=args.iterations)
+    else:
+        result = {"error": "Provide --trace-id"}
+
+    if args.json:
+        print(json.dumps(result, indent=2, default=str))
+    else:
+        print(json.dumps(result, indent=2, default=str))
+
+
+if __name__ == "__main__":
+    main()