devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/metrics/session_writer.py

@@ -0,0 +1,81 @@
+"""
+Session metrics writer/reader for DevTorch.
+
+Session metrics capture per-session LLM usage signals.
+Stored at .GCC/metrics/session/<session_id>.json.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Optional
+
+
+_SESSION_DIR = "metrics/session"
+
+
+def write_session_metrics(gcc_dir: Path, session_id: str, data: dict) -> Path:
+    """
+    Write session metrics to .GCC/metrics/session/<session_id>.json.
+
+    Expected data fields:
+      tokens_used            — int, total tokens consumed this session
+      tokens_saved           — int, estimated savings (default: tokens_used × 0.3)
+      coverage               — float, ratio of committed decisions to total tool calls
+      confidence_distribution — dict mapping tier name → count
+      cold_start_ms          — float, cold-start latency in milliseconds
+
+    Missing optional fields are computed or defaulted. Returns the path written.
+    """
+    gcc_dir = Path(gcc_dir)
+    session_dir = gcc_dir / _SESSION_DIR
+    session_dir.mkdir(parents=True, exist_ok=True)
+
+    tokens_used = data.get("tokens_used", 0)
+    tokens_saved = data.get("tokens_saved", int(tokens_used * 0.3))
+
+    full_history = data.get("full_history_tokens", 0)
+    bundle = data.get("bundle_tokens", 0)
+    latency_speedup = data.get("latency_speedup_factor", 0.0)
+    if latency_speedup == 0.0 and bundle > 0:
+        latency_speedup = round(full_history / bundle, 4)
+
+    record = {
+        "session_id": session_id,
+        "tokens_used": tokens_used,
+        "tokens_saved": tokens_saved,
+        "bundle_tokens": bundle,
+        "full_history_tokens": full_history,
+        "latency_speedup_factor": latency_speedup,
+        "developer_id": data.get("developer_id"),
+        "coverage": data.get("coverage", 0.0),
+        "confidence_distribution": data.get("confidence_distribution", {}),
+        "cold_start_ms": data.get("cold_start_ms", 0.0),
+    }
+    if "created_at" in data:
+        record["created_at"] = data["created_at"]
+
+    out_path = session_dir / f"{session_id}.json"
+    out_path.write_text(json.dumps(record, indent=2, sort_keys=True), encoding="utf-8")
+    return out_path
+
+
+def read_session_metrics(gcc_dir: Path, session_id: str) -> Optional[dict]:
+    """
+    Read session metrics for *session_id*.
+
+    Returns the dict if found, None if missing or unreadable.
+    """
+    gcc_dir = Path(gcc_dir)
+    path = gcc_dir / _SESSION_DIR / f"{session_id}.json"
+    if not path.exists():
+        return None
+    try:
+        text = path.read_text(encoding="utf-8")
+        data = json.loads(text)
+        if not isinstance(data, dict):
+            return None
+        return data
+    except (json.JSONDecodeError, OSError):
+        return None

devtorch_core/metrics/shadow_ai.py

@@ -0,0 +1,117 @@
+"""
+Shadow AI detector for DevTorch.
+
+Compares git commit count to GCC COMMIT event count over the last 30 days.
+A high unregistered commit ratio suggests developers are bypassing GCC — a
+"shadow AI" pattern where AI-assisted commits skip governance logging.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from datetime import datetime, timezone, timedelta
+from pathlib import Path
+
+
+_EVENT_LOG = "events.log.jsonl"
+_THIRTY_DAYS_SECONDS = 30 * 24 * 3600
+
+
+def _utcnow() -> datetime:
+    return datetime.now(tz=timezone.utc)
+
+
+def _git_commit_count(repo_root: Path) -> int:
+    """Count git commits in the last 30 days. Returns 0 on any error."""
+    try:
+        result = subprocess.run(
+            ["git", "-C", str(repo_root), "log", "--oneline", "--since=30.days.ago"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if result.returncode != 0:
+            return 0
+        lines = [l for l in result.stdout.splitlines() if l.strip()]
+        return len(lines)
+    except Exception:
+        return 0
+
+
+def _gcc_commit_count(gcc_dir: Path) -> int:
+    """Count COMMIT events in events.log.jsonl in the last 30 days. Returns 0 on any error."""
+    event_log = gcc_dir / _EVENT_LOG
+    if not event_log.exists():
+        return 0
+    cutoff = _utcnow() - timedelta(seconds=_THIRTY_DAYS_SECONDS)
+    count = 0
+    try:
+        with event_log.open(encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    event = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                if event.get("event_type") != "COMMIT":
+                    continue
+                ts_str = event.get("timestamp", "")
+                try:
+                    # Parse ISO 8601 timestamp
+                    ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
+                    if ts.tzinfo is None:
+                        ts = ts.replace(tzinfo=timezone.utc)
+                    if ts >= cutoff:
+                        count += 1
+                except (ValueError, AttributeError):
+                    # If we can't parse timestamp, include the event conservatively
+                    count += 1
+    except OSError:
+        return 0
+    return count
+
+
+def detect_shadow_ai(repo_root: Path) -> dict:
+    """
+    Compare git commit count to GCC COMMIT event count over the last 30 days.
+
+    Returns:
+      {
+        "git_commits":   int   — total git commits in last 30 days
+        "gcc_commits":   int   — GCC COMMIT events in last 30 days
+        "unregistered":  int   — git_commits - gcc_commits (floor 0)
+        "shadow_ratio":  float — unregistered / git_commits (0.0 if no git commits)
+        "flagged":       bool  — True when unregistered > 3 and shadow_ratio > 0.3
+      }
+
+    All errors return safe defaults (0 counts, flagged=False).
+    """
+    try:
+        repo_root = Path(repo_root)
+        gcc_dir = repo_root / ".GCC"
+
+        git_commits = _git_commit_count(repo_root)
+        gcc_commits = _gcc_commit_count(gcc_dir)
+
+        unregistered = max(0, git_commits - gcc_commits)
+        shadow_ratio = unregistered / git_commits if git_commits > 0 else 0.0
+        flagged = unregistered > 3 and shadow_ratio > 0.3
+
+        return {
+            "git_commits": git_commits,
+            "gcc_commits": gcc_commits,
+            "unregistered": unregistered,
+            "shadow_ratio": shadow_ratio,
+            "flagged": flagged,
+        }
+    except Exception:
+        return {
+            "git_commits": 0,
+            "gcc_commits": 0,
+            "unregistered": 0,
+            "shadow_ratio": 0.0,
+            "flagged": False,
+        }

devtorch_core/metrics/sprint_writer.py

@@ -0,0 +1,243 @@
+"""
+Sprint metrics writer/reader for DevTorch.
+
+Sprint metrics capture per-sprint governance signals.
+Stored at .GCC/metrics/sprint/<sprint_id>.json.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import json
+import math
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from devtorch_core.metrics.mcs import compute_mcs, CollisionEvent
+
+
+_SPRINT_DIR = "metrics/sprint"
+
+
+def _read_events_log(gcc_dir: Path) -> List[Dict[str, Any]]:
+    """Read all events from events.log.jsonl; return empty list on any error."""
+    log_file = gcc_dir / "events.log.jsonl"
+    events: List[Dict[str, Any]] = []
+    if not log_file.exists():
+        return events
+    try:
+        with log_file.open("r", encoding="utf-8") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    events.append(json.loads(line))
+                except json.JSONDecodeError:
+                    continue
+    except OSError:
+        pass
+    return events
+
+
+def _read_commit_count(gcc_dir: Path) -> int:
+    """Count commit files in .GCC/commits."""
+    commits_dir = gcc_dir / "commits"
+    if not commits_dir.exists():
+        return 0
+    return sum(1 for p in commits_dir.iterdir() if p.suffix == ".json")
+
+
+def _read_branch_count(gcc_dir: Path) -> int:
+    """Count branch ref files in .GCC/refs/branches."""
+    branches_dir = gcc_dir / "refs" / "branches"
+    if not branches_dir.exists():
+        return 0
+    return sum(1 for p in branches_dir.iterdir() if p.is_file())
+
+
+def _read_theta_collisions(gcc_dir: Path) -> List[CollisionEvent]:
+    """Detect collisions from theta.json where source_nodes diverge (std_dev > 0.2)."""
+    theta_file = gcc_dir / "theta.json"
+    if not theta_file.exists():
+        return []
+    try:
+        data = json.loads(theta_file.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return []
+
+    cv: Dict[str, Any] = data.get("coordination_vector", {})
+    collisions: List[CollisionEvent] = []
+    for concept, info in cv.items():
+        source_nodes: List[Dict[str, Any]] = info.get("source_nodes", [])
+        if len(source_nodes) < 2:
+            continue
+        confidences = [float(n.get("confidence", 0.0)) for n in source_nodes]
+        mean_c = sum(confidences) / len(confidences)
+        variance = sum((c - mean_c) ** 2 for c in confidences) / len(confidences)
+        std_dev = math.sqrt(variance)
+        if std_dev <= 0.2:
+            continue
+        severity = 0.8 if std_dev > 0.4 else 0.5
+        collisions.append(
+            CollisionEvent(
+                timestamp=info.get("last_updated", "") or "",
+                branch_a=str(source_nodes[0].get("branch", source_nodes[0].get("agent_id", "a"))),
+                branch_b=str(source_nodes[1].get("branch", source_nodes[1].get("agent_id", "b"))),
+                concept=concept,
+                severity=severity,
+                resolution_minutes=0.0,
+            )
+        )
+    return collisions
+
+
+def _count_i3_violations(events: List[Dict[str, Any]]) -> int:
+    """Count I3 invariant violations, falling back to any invariant violation."""
+    i3 = sum(
+        1
+        for ev in events
+        if ev.get("event_type", ev.get("type", "")) == "INVARIANT_VIOLATION"
+        and "I3" in str(ev.get("invariant", ev.get("detail", "")))
+    )
+    if i3 == 0:
+        i3 = sum(
+            1
+            for ev in events
+            if ev.get("event_type", ev.get("type", "")) == "INVARIANT_VIOLATION"
+        )
+    return i3
+
+
+def _decision_rates(events: List[Dict[str, Any]]) -> tuple[float, float]:
+    """Return (acceptance_rate, override_rate) from decision-like events."""
+    decision_types: set[str] = {"ACCEPT", "OVERRIDE", "HITL_REQUEST", "REJECT"}
+    decisions: List[Dict[str, Any]] = []
+    for ev in events:
+        etype = ev.get("event_type", ev.get("type", "")).upper()
+        if etype in decision_types or any(t in etype for t in decision_types):
+            decisions.append(ev)
+    if not decisions:
+        return 1.0, 0.0
+    accepted = sum(
+        1 for d in decisions if "ACCEPT" in d.get("event_type", d.get("type", "")).upper()
+    )
+    overridden = sum(
+        1 for d in decisions if "OVERRIDE" in d.get("event_type", d.get("type", "")).upper()
+    )
+    n = len(decisions)
+    return round(accepted / n, 4), round(overridden / n, 4)
+
+
+def compute_sprint_metrics(gcc_dir: Path, sprint_id: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Compute sprint metrics from the current .GCC/ state and persist them.
+
+    Metrics computed:
+      mcs_score              — from theta.json source-node divergence
+      i3_violations          — count of I3 invariant failures
+      collisions_prevented   — count of collisions detected (proxy for prevented)
+      override_rate          — fraction of overridden AI suggestions
+      acceptance_rate        — fraction of accepted AI suggestions
+      branch_count           — number of active branches
+      commit_count           — number of commits
+
+    If *sprint_id* is None, a default id of the form "sprint_YYYY-MM-DD" is used.
+    Returns the metrics dict (not the Pydantic model).
+    """
+    gcc_dir = Path(gcc_dir)
+    events = _read_events_log(gcc_dir)
+    collisions = _read_theta_collisions(gcc_dir)
+
+    mcs_score = compute_mcs(collisions)
+    i3_violations = _count_i3_violations(events)
+    collisions_prevented = len(collisions) + sum(
+        1 for ev in events if ev.get("event_type", ev.get("type", "")) == "VARIANCE_ALERT"
+    )
+    acceptance_rate, override_rate = _decision_rates(events)
+    branch_count = _read_branch_count(gcc_dir)
+    commit_count = _read_commit_count(gcc_dir)
+
+    if sprint_id is None:
+        sprint_id = f"sprint_{_dt.date.today().isoformat()}"
+
+    record = {
+        "sprint_id": sprint_id,
+        "mcs_score": mcs_score,
+        "i3_violations": i3_violations,
+        "collisions_prevented": collisions_prevented,
+        "override_rate": override_rate,
+        "acceptance_rate": acceptance_rate,
+        "branch_count": branch_count,
+        "commit_count": commit_count,
+    }
+
+    sprint_dir = gcc_dir / _SPRINT_DIR
+    sprint_dir.mkdir(parents=True, exist_ok=True)
+    out_path = sprint_dir / f"{sprint_id}.json"
+    out_path.write_text(json.dumps(record, indent=2, sort_keys=True), encoding="utf-8")
+    return record
+
+
+def write_sprint_metrics(gcc_dir: Path, sprint_id: str, data: dict) -> Path:
+    """
+    Write sprint metrics to .GCC/metrics/sprint/<sprint_id>.json.
+
+    Expected data fields:
+      mcs_score              — float [0,1]; computed via compute_mcs if not provided
+      i3_violations          — int, count of I3 invariant failures this sprint
+      collisions_prevented   — int, estimated collisions prevented by GCC
+      override_rate          — float [0,1], fraction of AI suggestions overridden
+      acceptance_rate        — float [0,1], fraction of AI suggestions accepted
+      branch_count           — int, number of branches active this sprint
+      commit_count           — int, number of commits this sprint
+
+    If mcs_score is not provided, attempts to compute it from an empty event list
+    (returns 0.0). Returns the path written.
+    """
+    gcc_dir = Path(gcc_dir)
+    sprint_dir = gcc_dir / _SPRINT_DIR
+    sprint_dir.mkdir(parents=True, exist_ok=True)
+
+    if "mcs_score" not in data:
+        try:
+            mcs_score = compute_mcs([])
+        except Exception:
+            mcs_score = 0.0
+    else:
+        mcs_score = float(data["mcs_score"])
+
+    record = {
+        "sprint_id": sprint_id,
+        "mcs_score": mcs_score,
+        "i3_violations": int(data.get("i3_violations", 0)),
+        "collisions_prevented": int(data.get("collisions_prevented", 0)),
+        "override_rate": float(data.get("override_rate", 0.0)),
+        "acceptance_rate": float(data.get("acceptance_rate", 0.0)),
+        "branch_count": int(data.get("branch_count", 0)),
+        "commit_count": int(data.get("commit_count", 0)),
+    }
+
+    out_path = sprint_dir / f"{sprint_id}.json"
+    out_path.write_text(json.dumps(record, indent=2, sort_keys=True), encoding="utf-8")
+    return out_path
+
+
+def read_sprint_metrics(gcc_dir: Path, sprint_id: str) -> Optional[dict]:
+    """
+    Read sprint metrics for *sprint_id*.
+
+    Returns the dict if found, None if missing or unreadable.
+    """
+    gcc_dir = Path(gcc_dir)
+    path = gcc_dir / _SPRINT_DIR / f"{sprint_id}.json"
+    if not path.exists():
+        return None
+    try:
+        text = path.read_text(encoding="utf-8")
+        data = json.loads(text)
+        if not isinstance(data, dict):
+            return None
+        return data
+    except (json.JSONDecodeError, OSError):
+        return None

devtorch_core/observability/__init__.py

@@ -0,0 +1,78 @@
+"""
+devtorch_core.observability — Structured governance reporting and enterprise metrics.
+
+Public API:
+  - ObservabilityReport, build_report        (report.py)
+  - format_report_markdown, format_report_json  (formatter.py)
+  - WebhookPayload, send_webhook, send_metrics_if_configured  (webhook.py)
+"""
+
+from .report import (
+    ObservabilityReport,
+    ReportTimeRange,
+    build_report,
+    report_to_dict,
+)
+from .formatter import (
+    format_report_markdown,
+    format_report_json,
+)
+from .webhook import (
+    WebhookConfig,
+    WebhookPayload,
+    WebhookResult,
+    load_webhook_config,
+    build_webhook_payload,
+    send_webhook,
+    send_metrics_if_configured,
+)
+from .datadog import (
+    DatadogConfig,
+    load_datadog_config,
+    send_datadog_metrics,
+)
+from .splunk import (
+    SplunkConfig,
+    load_splunk_config,
+    send_to_splunk,
+    stream_events_log,
+)
+from .servicenow import (
+    ServiceNowConfig,
+    load_servicenow_config,
+    build_change_record,
+    create_servicenow_change,
+)
+
+__all__ = [
+    # report
+    "ObservabilityReport",
+    "ReportTimeRange",
+    "build_report",
+    "report_to_dict",
+    # formatter
+    "format_report_markdown",
+    "format_report_json",
+    # webhook
+    "WebhookConfig",
+    "WebhookPayload",
+    "WebhookResult",
+    "load_webhook_config",
+    "build_webhook_payload",
+    "send_webhook",
+    "send_metrics_if_configured",
+    # datadog
+    "DatadogConfig",
+    "load_datadog_config",
+    "send_datadog_metrics",
+    # splunk
+    "SplunkConfig",
+    "load_splunk_config",
+    "send_to_splunk",
+    "stream_events_log",
+    # servicenow
+    "ServiceNowConfig",
+    "load_servicenow_config",
+    "build_change_record",
+    "create_servicenow_change",
+]

devtorch_core/observability/datadog.py

@@ -0,0 +1,157 @@
+"""
+Datadog metrics exporter for DevTorch observability reports.
+
+Sends metrics-only time-series points to the Datadog `/api/v1/series` endpoint.
+NEVER includes: prompts, code, reasoning tokens, commit message bodies,
+sensitivity signal text, or any LLM response content.
+
+Only sends: counts, scores, timestamps, and branch/state metadata.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+import time
+import urllib.request
+import urllib.error
+from typing import Any, Dict, List, Optional
+
+from .report import ObservabilityReport
+from .webhook import WebhookResult
+
+
+# ---------------------------------------------------------------------------
+# Dataclasses
+# ---------------------------------------------------------------------------
+
+@dataclasses.dataclass
+class DatadogConfig:
+    """Configuration for Datadog metrics export."""
+    api_key: str
+    site: str = "datadoghq.com"
+    timeout_seconds: int = 5
+    enabled: bool = True
+    prefix: str = "devtorch"
+
+
+# ---------------------------------------------------------------------------
+# Config loader
+# ---------------------------------------------------------------------------
+
+def load_datadog_config() -> Optional[DatadogConfig]:
+    """
+    Load Datadog configuration from environment variables.
+
+    Environment variables:
+      - DEVTORCH_DATADOG_API_KEY (required)
+      - DEVTORCH_DATADOG_SITE
+      - DEVTORCH_DATADOG_TIMEOUT_SECONDS
+
+    Returns None if DEVTORCH_DATADOG_API_KEY is not set.
+    Never raises.
+    """
+    try:
+        api_key = os.environ.get("DEVTORCH_DATADOG_API_KEY", "").strip()
+        if not api_key:
+            return None
+        site = os.environ.get("DEVTORCH_DATADOG_SITE", "datadoghq.com").strip() or "datadoghq.com"
+        timeout_str = os.environ.get("DEVTORCH_DATADOG_TIMEOUT_SECONDS", "5").strip()
+        try:
+            timeout_seconds = int(timeout_str)
+        except (ValueError, TypeError):
+            timeout_seconds = 5
+        return DatadogConfig(
+            api_key=api_key,
+            site=site,
+            timeout_seconds=timeout_seconds,
+        )
+    except Exception:
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Metrics builder
+# ---------------------------------------------------------------------------
+
+def _build_series_payload(report: ObservabilityReport) -> Dict[str, Any]:
+    """
+    Build a Datadog `/api/v1/series` payload from an ObservabilityReport.
+
+    Only includes metadata metrics: counts, scores, and timestamp fields.
+    """
+    timestamp = int(time.time())
+    prefix = "devtorch"
+
+    rdp = report.rdp_budget
+    rdp_epsilon_used = float(rdp.get("epsilon_used", 0.0))
+
+    # Disclosure health score: 1.0 when no high-disclosure events, lower otherwise.
+    sensitivity_total = max(report.sensitivity_event_count, 1)
+    dhs_score = 1.0 - (report.high_disclosure_count / sensitivity_total)
+
+    # Metadata capability score: simple binary signal from textual mode gating.
+    cap = report.capability_state
+    textual_allowed = bool(cap.get("textual_mode_allowed", False))
+    mcs_score = 1.0 if textual_allowed else 0.0
+
+    series: List[Dict[str, Any]] = [
+        {"metric": f"{prefix}.commit_count", "points": [[timestamp, report.commit_count]], "type": "gauge"},
+        {"metric": f"{prefix}.sensitivity_event_count", "points": [[timestamp, report.sensitivity_event_count]], "type": "gauge"},
+        {"metric": f"{prefix}.high_disclosure_count", "points": [[timestamp, report.high_disclosure_count]], "type": "gauge"},
+        {"metric": f"{prefix}.branch_count", "points": [[timestamp, report.branch_count]], "type": "gauge"},
+        {"metric": f"{prefix}.sis_quarantine_count", "points": [[timestamp, report.sis_quarantine_count]], "type": "gauge"},
+        {"metric": f"{prefix}.mcs_score", "points": [[timestamp, mcs_score]], "type": "gauge"},
+        {"metric": f"{prefix}.dhs_score", "points": [[timestamp, dhs_score]], "type": "gauge"},
+        {"metric": f"{prefix}.collision_count", "points": [[timestamp, 0]], "type": "gauge"},
+        {"metric": f"{prefix}.theta_concept_count", "points": [[timestamp, len(report.theta_top_concepts)]], "type": "gauge"},
+        {"metric": f"{prefix}.rdp_epsilon_used", "points": [[timestamp, rdp_epsilon_used]], "type": "gauge"},
+    ]
+
+    return {"series": series}
+
+
+# ---------------------------------------------------------------------------
+# Sender
+# ---------------------------------------------------------------------------
+
+def send_datadog_metrics(report: ObservabilityReport, config: DatadogConfig) -> WebhookResult:
+    """
+    POST an ObservabilityReport as Datadog time-series metrics.
+
+    Uses urllib.request — no external dependencies.
+    Never raises — returns WebhookResult with success=False on any error.
+
+    Headers sent:
+      DD-API-KEY: {api_key}
+      Content-Type: application/json
+    """
+    if not config.enabled:
+        return WebhookResult(success=False, status_code=0, error="Datadog export is disabled")
+
+    try:
+        body_dict = _build_series_payload(report)
+        body_bytes = json.dumps(body_dict).encode("utf-8")
+
+        url = f"https://api.{config.site}/api/v1/series"
+        req = urllib.request.Request(
+            url=url,
+            data=body_bytes,
+            method="POST",
+            headers={
+                "DD-API-KEY": config.api_key,
+                "Content-Type": "application/json",
+            },
+        )
+
+        with urllib.request.urlopen(req, timeout=config.timeout_seconds) as resp:
+            status_code = resp.status
+            return WebhookResult(success=True, status_code=status_code, error="")
+
+    except urllib.error.HTTPError as exc:
+        return WebhookResult(success=False, status_code=exc.code, error=str(exc))
+    except urllib.error.URLError as exc:
+        return WebhookResult(success=False, status_code=0, error=str(exc))
+    except Exception as exc:
+        return WebhookResult(success=False, status_code=0, error=str(exc))