devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/observability/splunk.py

@@ -0,0 +1,218 @@
+"""
+Splunk SIEM integration for DevTorch governance events.
+
+Streams `.GCC/events.log.jsonl` to the Splunk HTTP Event Collector (HEC)
+endpoint `/services/collector/event` in batches.
+
+NEVER includes: prompts, code, reasoning tokens, commit message bodies,
+sensitivity signal text, or any LLM response content.
+
+Events containing forbidden keys are dropped before sending.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+import urllib.request
+import urllib.error
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional
+
+from .webhook import WebhookResult
+
+
+# ---------------------------------------------------------------------------
+# Forbidden content keys (privacy guard)
+# ---------------------------------------------------------------------------
+
+_FORBIDDEN_KEYS = frozenset({
+    "prompt",
+    "content",
+    "messages",
+    "reasoning",
+    "text",
+    "code",
+})
+
+
+# ---------------------------------------------------------------------------
+# Dataclasses
+# ---------------------------------------------------------------------------
+
+@dataclasses.dataclass
+class SplunkConfig:
+    """Configuration for Splunk HEC ingestion."""
+    hec_url: str
+    hec_token: str
+    source: str = "devtorch"
+    sourcetype: str = "_json"
+    timeout_seconds: int = 5
+    batch_size: int = 100
+    enabled: bool = True
+
+
+# ---------------------------------------------------------------------------
+# Config loader
+# ---------------------------------------------------------------------------
+
+def load_splunk_config() -> Optional[SplunkConfig]:
+    """
+    Load Splunk configuration from environment variables.
+
+    Environment variables:
+      - DEVTORCH_SPLUNK_HEC_URL (required)
+      - DEVTORCH_SPLUNK_HEC_TOKEN (required)
+      - DEVTORCH_SPLUNK_SOURCE
+
+    Returns None if required variables are missing.
+    Never raises.
+    """
+    try:
+        hec_url = os.environ.get("DEVTORCH_SPLUNK_HEC_URL", "").strip()
+        hec_token = os.environ.get("DEVTORCH_SPLUNK_HEC_TOKEN", "").strip()
+        if not hec_url or not hec_token:
+            return None
+        source = os.environ.get("DEVTORCH_SPLUNK_SOURCE", "devtorch").strip() or "devtorch"
+        return SplunkConfig(
+            hec_url=hec_url,
+            hec_token=hec_token,
+            source=source,
+        )
+    except Exception:
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Event streaming
+# ---------------------------------------------------------------------------
+
+def stream_events_log(gcc_dir: Path) -> Iterator[Dict[str, Any]]:
+    """
+    Yield parsed JSON objects from `.GCC/events.log.jsonl`.
+
+    Skips malformed lines silently. Never raises.
+    """
+    event_log = gcc_dir / "events.log.jsonl"
+    if not event_log.exists():
+        return
+
+    try:
+        with event_log.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    yield json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+    except Exception:
+        return
+
+
+# ---------------------------------------------------------------------------
+# Privacy guard
+# ---------------------------------------------------------------------------
+
+def _event_contains_forbidden_keys(event: Dict[str, Any]) -> bool:
+    """Return True if the event payload contains any forbidden content keys."""
+    return not _FORBIDDEN_KEYS.isdisjoint(event.keys())
+
+
+# ---------------------------------------------------------------------------
+# Sender
+# ---------------------------------------------------------------------------
+
+def send_to_splunk(gcc_dir: Path, config: SplunkConfig) -> WebhookResult:
+    """
+    Send governance events from `.GCC/events.log.jsonl` to Splunk HEC.
+
+    Events are sent in batches of up to config.batch_size. Events containing
+    forbidden keys (`prompt`, `content`, `messages`, `reasoning`, `text`, `code`)
+    are skipped.
+
+    Uses urllib.request — no external dependencies.
+    Never raises — returns WebhookResult with success=False on any error.
+
+    Headers sent:
+      Authorization: Splunk {hec_token}
+      Content-Type: application/json
+    """
+    if not config.enabled:
+        return WebhookResult(success=False, status_code=0, error="Splunk export is disabled")
+
+    try:
+        url = f"{config.hec_url.rstrip('/')}/services/collector/event"
+        headers = {
+            "Authorization": f"Splunk {config.hec_token}",
+            "Content-Type": "application/json",
+        }
+
+        batch: List[Dict[str, Any]] = []
+        total_sent = 0
+        last_status = 200
+        last_error = ""
+
+        for event in stream_events_log(gcc_dir):
+            if _event_contains_forbidden_keys(event):
+                continue
+
+            # Wrap each event in Splunk HEC envelope.
+            wrapped = {
+                "source": config.source,
+                "sourcetype": config.sourcetype,
+                "event": event,
+            }
+            batch.append(wrapped)
+
+            if len(batch) >= config.batch_size:
+                result = _send_batch(url, headers, batch, config.timeout_seconds)
+                if result.success:
+                    total_sent += len(batch)
+                else:
+                    last_status = result.status_code
+                    last_error = result.error
+                batch = []
+
+        if batch:
+            result = _send_batch(url, headers, batch, config.timeout_seconds)
+            if result.success:
+                total_sent += len(batch)
+            else:
+                last_status = result.status_code
+                last_error = result.error
+
+        if last_error:
+            return WebhookResult(success=False, status_code=last_status, error=last_error)
+        return WebhookResult(success=True, status_code=last_status, error="")
+
+    except Exception as exc:
+        return WebhookResult(success=False, status_code=0, error=str(exc))
+
+
+def _send_batch(url: str, headers: Dict[str, str], batch: List[Dict[str, Any]], timeout: int) -> WebhookResult:
+    """POST a batch of Splunk HEC events."""
+    try:
+        # Splunk HEC accepts newline-separated JSON payloads.
+        body_lines = [json.dumps(item) for item in batch]
+        body_bytes = "\n".join(body_lines).encode("utf-8")
+
+        req = urllib.request.Request(
+            url=url,
+            data=body_bytes,
+            method="POST",
+            headers=headers,
+        )
+
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            status_code = resp.status
+            return WebhookResult(success=True, status_code=status_code, error="")
+
+    except urllib.error.HTTPError as exc:
+        return WebhookResult(success=False, status_code=exc.code, error=str(exc))
+    except urllib.error.URLError as exc:
+        return WebhookResult(success=False, status_code=0, error=str(exc))
+    except Exception as exc:
+        return WebhookResult(success=False, status_code=0, error=str(exc))

devtorch_core/observability/webhook.py

@@ -0,0 +1,227 @@
+"""
+Enterprise metrics webhook sender.
+
+Sends a METRICS-ONLY payload to a configured endpoint.
+NEVER includes: prompts, code, reasoning tokens, commit message bodies,
+sensitivity signal text, or any LLM response content.
+
+Only sends: counts, scores, timestamps, branch names, node state.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+import urllib.request
+import urllib.error
+from typing import Optional
+
+from .report import ObservabilityReport
+
+
+# ---------------------------------------------------------------------------
+# Approved metric keys sent in the webhook payload (explicit allowlist)
+# ---------------------------------------------------------------------------
+
+_PAYLOAD_KEYS = frozenset({
+    "org_id",
+    "node_state",
+    "branch",
+    "commit_count",
+    "sensitivity_count",
+    "high_disclosure_count",
+    "dhs_score",
+    "mcs_score",
+    "rdp_epsilon_used",
+    "rdp_exhausted",
+    "variance_alert_count",
+    "generated_at",
+})
+
+
+# ---------------------------------------------------------------------------
+# Dataclasses
+# ---------------------------------------------------------------------------
+
+@dataclasses.dataclass
+class WebhookConfig:
+    """Configuration for the enterprise metrics webhook."""
+    endpoint_url: str   # DEVTORCH_METRICS_WEBHOOK_URL
+    org_id: str         # DEVTORCH_ORG_ID
+    api_key: str        # DEVTORCH_METRICS_API_KEY
+    timeout_seconds: int = 10
+
+
+@dataclasses.dataclass
+class WebhookPayload:
+    """
+    Metrics-only payload sent to the webhook endpoint.
+
+    Explicit field list — every field is named here, nothing else.
+    Contains: counts, scores, timestamps, branch name, node state only.
+    """
+    org_id: str
+    node_state: str
+    branch: str
+    commit_count: int
+    sensitivity_count: int
+    high_disclosure_count: int
+    dhs_score: Optional[float]
+    mcs_score: Optional[float]
+    rdp_epsilon_used: float
+    rdp_exhausted: bool
+    variance_alert_count: int
+    generated_at: str
+
+
+@dataclasses.dataclass
+class WebhookResult:
+    """Result of a webhook POST attempt."""
+    success: bool
+    status_code: int
+    error: str
+
+
+# ---------------------------------------------------------------------------
+# Config loader
+# ---------------------------------------------------------------------------
+
+def load_webhook_config() -> Optional[WebhookConfig]:
+    """
+    Load webhook configuration from environment variables.
+
+    Returns None if DEVTORCH_METRICS_WEBHOOK_URL is not set.
+    Never raises.
+    """
+    try:
+        endpoint_url = os.environ.get("DEVTORCH_METRICS_WEBHOOK_URL", "").strip()
+        if not endpoint_url:
+            return None
+        org_id = os.environ.get("DEVTORCH_ORG_ID", "").strip()
+        api_key = os.environ.get("DEVTORCH_METRICS_API_KEY", "").strip()
+        timeout_str = os.environ.get("DEVTORCH_METRICS_TIMEOUT_SECONDS", "10").strip()
+        try:
+            timeout_seconds = int(timeout_str)
+        except (ValueError, TypeError):
+            timeout_seconds = 10
+        return WebhookConfig(
+            endpoint_url=endpoint_url,
+            org_id=org_id,
+            api_key=api_key,
+            timeout_seconds=timeout_seconds,
+        )
+    except Exception:
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Payload builder
+# ---------------------------------------------------------------------------
+
+def build_webhook_payload(report: ObservabilityReport, org_id: str) -> WebhookPayload:
+    """
+    Map ObservabilityReport fields to a WebhookPayload.
+
+    Explicit field mapping: every included field is named here, nothing else.
+    Intentionally omits: prompts, commit messages, sensitivity signal text,
+    theta concept names beyond counts, code, or any LLM output.
+    """
+    rdp = report.rdp_budget
+    rdp_epsilon_used = float(rdp.get("epsilon_used", 0.0))
+    rdp_epsilon_max = float(rdp.get("epsilon_max", 1.0))
+    rdp_exhausted = bool(rdp.get("read_only", False))
+
+    variance = report.variance_state
+    variance_alert_count = int(variance.get("alert_count", 0))
+
+    return WebhookPayload(
+        org_id=org_id,
+        node_state=report.node_state,
+        branch=report.branch,
+        commit_count=report.commit_count,
+        sensitivity_count=report.sensitivity_event_count,
+        high_disclosure_count=report.high_disclosure_count,
+        dhs_score=None,   # not computed here; callers may populate via separate DHS call
+        mcs_score=None,   # not computed here; callers may populate via separate MCS call
+        rdp_epsilon_used=rdp_epsilon_used,
+        rdp_exhausted=rdp_exhausted,
+        variance_alert_count=variance_alert_count,
+        generated_at=report.generated_at,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Webhook sender
+# ---------------------------------------------------------------------------
+
+def send_webhook(payload: WebhookPayload, config: WebhookConfig) -> WebhookResult:
+    """
+    POST the metrics payload to config.endpoint_url.
+
+    Uses urllib.request — no external dependencies.
+    Never raises — returns WebhookResult with success=False on any error.
+
+    Headers sent:
+      Authorization: Bearer {api_key}
+      Content-Type: application/json
+      X-DevTorch-Org: {org_id}
+    """
+    try:
+        body_dict = {
+            "org_id": payload.org_id,
+            "node_state": payload.node_state,
+            "branch": payload.branch,
+            "commit_count": payload.commit_count,
+            "sensitivity_count": payload.sensitivity_count,
+            "high_disclosure_count": payload.high_disclosure_count,
+            "dhs_score": payload.dhs_score,
+            "mcs_score": payload.mcs_score,
+            "rdp_epsilon_used": payload.rdp_epsilon_used,
+            "rdp_exhausted": payload.rdp_exhausted,
+            "variance_alert_count": payload.variance_alert_count,
+            "generated_at": payload.generated_at,
+        }
+        body_bytes = json.dumps(body_dict).encode("utf-8")
+
+        req = urllib.request.Request(
+            url=config.endpoint_url,
+            data=body_bytes,
+            method="POST",
+            headers={
+                "Authorization": f"Bearer {config.api_key}",
+                "Content-Type": "application/json",
+                "X-DevTorch-Org": config.org_id,
+            },
+        )
+
+        with urllib.request.urlopen(req, timeout=config.timeout_seconds) as resp:
+            status_code = resp.status
+            return WebhookResult(success=True, status_code=status_code, error="")
+
+    except urllib.error.HTTPError as exc:
+        return WebhookResult(success=False, status_code=exc.code, error=str(exc))
+    except urllib.error.URLError as exc:
+        return WebhookResult(success=False, status_code=0, error=str(exc))
+    except Exception as exc:
+        return WebhookResult(success=False, status_code=0, error=str(exc))
+
+
+# ---------------------------------------------------------------------------
+# Convenience helper
+# ---------------------------------------------------------------------------
+
+def send_metrics_if_configured(report: ObservabilityReport) -> Optional[WebhookResult]:
+    """
+    Load config from environment; if not configured return None.
+    Otherwise build payload, send webhook, and return WebhookResult.
+    Never raises.
+    """
+    try:
+        config = load_webhook_config()
+        if config is None:
+            return None
+        payload = build_webhook_payload(report, org_id=config.org_id)
+        return send_webhook(payload, config)
+    except Exception:
+        return None

devtorch_core/parser/__init__.py

@@ -0,0 +1,30 @@
+"""
+devtorch_core.parser
+====================
+Shared parser foundation for the S7 LLM Wrapper layer.
+
+Extracts structured data from LLM responses:
+
+* **blocks** — RACP XML blocks (``<devtorch:commit>``, ``<devtorch:sensitivity>``)
+* **thinking** — extended-thinking / reasoning content blocks
+* **inference** — heuristic fallback when no explicit RACP blocks were emitted
+"""
+from .blocks import extract_racp_blocks, strip_racp_blocks, CommitBlock, SensitivityBlock
+from .thinking import extract_thinking_blocks, thinking_to_commit_message, ThinkingBlock
+from .inference import infer_sensitivity, extract_decision_summary, InferredSensitivity
+
+__all__ = [
+    # blocks
+    "extract_racp_blocks",
+    "strip_racp_blocks",
+    "CommitBlock",
+    "SensitivityBlock",
+    # thinking
+    "extract_thinking_blocks",
+    "thinking_to_commit_message",
+    "ThinkingBlock",
+    # inference
+    "infer_sensitivity",
+    "extract_decision_summary",
+    "InferredSensitivity",
+]

devtorch_core/parser/blocks.py

@@ -0,0 +1,216 @@
+"""
+devtorch_core.parser.blocks
+===========================
+Extract and strip RACP-structured XML blocks emitted by LLMs that have been
+instructed with the DevTorch system prompt.
+
+LLM output formats handled
+---------------------------
+Self-closing tags::
+
+    <devtorch:commit message="migrate schema to v3" concepts="schema,migration"
+                     confidence="0.9"/>
+    <devtorch:sensitivity concept="schema_version"
+                          signal="migration collapses if format changes"
+                          confidence="0.91"
+                          disclosure="PROTECTED"/>
+
+Strategy
+--------
+1. Regex locates the raw tag strings in the text (tolerates attribute-order
+   variation and minor whitespace differences).
+2. ``xml.etree.ElementTree`` parses each located string for attribute values.
+3. Attribute values are validated at parse time: confidence is clamped to
+   [0, 1], disclosure is checked against the controlled vocabulary, and
+   required attributes (message, concept, signal) must be present.
+4. Malformed or invalid tags are logged and skipped — never raised.
+"""
+from __future__ import annotations
+
+import logging
+import re
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+
+logger = logging.getLogger(__name__)
+
+# Controlled vocabulary for disclosure levels
+VALID_DISCLOSURE_LEVELS = frozenset({"PUBLIC", "PROTECTED", "PRIVATE"})
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+@dataclass
+class CommitBlock:
+    message: str
+    confidence: float = 0.9
+    concepts: list[str] = field(default_factory=list)
+
+
+@dataclass
+class SensitivityBlock:
+    concept: str
+    signal: str
+    confidence: float = 0.9
+    disclosure: str = "PROTECTED"
+
+
+# ---------------------------------------------------------------------------
+# Internal regex patterns
+# ---------------------------------------------------------------------------
+
+# Matches self-closing <devtorch:commit ... /> tags.
+_RE_COMMIT = re.compile(
+    r"<devtorch:commit\b([^>]*?)/>",
+    re.DOTALL | re.IGNORECASE,
+)
+
+# Matches self-closing <devtorch:sensitivity ... /> tags.
+_RE_SENSITIVITY = re.compile(
+    r"<devtorch:sensitivity\b([^>]*?)/>",
+    re.DOTALL | re.IGNORECASE,
+)
+
+# Matches any paired <devtorch:*>...</devtorch:*> tags (for strip).
+_RE_PAIRED = re.compile(
+    r"<devtorch:[^>]+>.*?</devtorch:[^>]+>",
+    re.DOTALL | re.IGNORECASE,
+)
+
+# Matches all self-closing devtorch tags (for strip).
+_RE_SELF_CLOSING = re.compile(
+    r"<devtorch:\w[\w:.-]*\b[^>]*?/>",
+    re.DOTALL | re.IGNORECASE,
+)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _parse_tag(raw: str) -> ET.Element | None:
+    """
+    Parse a raw XML tag string via ElementTree.
+
+    The tag may use namespace prefixes (``devtorch:commit``) which are not
+    valid bare XML without a namespace declaration.  We normalise the prefix
+    to a plain element name so ElementTree can handle it.
+
+    Returns the parsed Element, or None on error.
+    """
+    # Normalise namespace prefix: devtorch:foo -> devtorch_foo
+    normalised = re.sub(r"<(/?)\s*devtorch:", r"<\1devtorch_", raw)
+    try:
+        return ET.fromstring(normalised)
+    except ET.ParseError as exc:
+        logger.warning("devtorch parser: malformed tag skipped — %s | raw=%r", exc, raw)
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def extract_racp_blocks(
+    text: str,
+) -> tuple[list[CommitBlock], list[SensitivityBlock]]:
+    """
+    Extract all ``<devtorch:commit>`` and ``<devtorch:sensitivity>`` blocks
+    from *text*.
+
+    Uses regex to locate tag boundaries, then ``xml.etree.ElementTree`` to
+    parse attribute values.  Attribute order variations and minor whitespace
+    are handled gracefully.  Malformed tags are logged and skipped.
+
+    Returns
+    -------
+    (commit_blocks, sensitivity_blocks)
+    """
+    commits: list[CommitBlock] = []
+    sensitivities: list[SensitivityBlock] = []
+
+    # --- commit blocks ---
+    for match in _RE_COMMIT.finditer(text):
+        raw_tag = match.group(0)
+        elem = _parse_tag(raw_tag)
+        if elem is None:
+            continue
+        message = elem.get("message")
+        if not message:
+            logger.warning("devtorch parser: <devtorch:commit> missing 'message' attribute — skipped | raw=%r", raw_tag)
+            continue
+        try:
+            confidence = float(elem.get("confidence", 0.9))
+            if not 0.0 <= confidence <= 1.0:
+                logger.warning(
+                    "devtorch parser: commit confidence %.2f out of range [0,1] — skipped | raw=%r",
+                    confidence, raw_tag,
+                )
+                continue
+        except (ValueError, TypeError):
+            logger.warning("devtorch parser: invalid confidence value in commit block — defaulting to 0.9")
+            confidence = 0.9
+        concepts_raw = elem.get("concepts") or elem.get("concepts_used") or ""
+        concepts = [c.strip() for c in concepts_raw.split(",") if c.strip()]
+        commits.append(CommitBlock(message=message, confidence=confidence, concepts=concepts))
+
+    # --- sensitivity blocks ---
+    for match in _RE_SENSITIVITY.finditer(text):
+        raw_tag = match.group(0)
+        elem = _parse_tag(raw_tag)
+        if elem is None:
+            continue
+        concept = elem.get("concept")
+        signal = elem.get("signal")
+        if not concept or not signal:
+            logger.warning(
+                "devtorch parser: <devtorch:sensitivity> missing required attributes — skipped | raw=%r",
+                raw_tag,
+            )
+            continue
+        try:
+            confidence = float(elem.get("confidence", 0.9))
+            if not 0.0 <= confidence <= 1.0:
+                logger.warning(
+                    "devtorch parser: sensitivity confidence %.2f out of range [0,1] — skipped | raw=%r",
+                    confidence, raw_tag,
+                )
+                continue
+        except (ValueError, TypeError):
+            logger.warning("devtorch parser: invalid confidence value in sensitivity block — defaulting to 0.9")
+            confidence = 0.9
+        disclosure = (elem.get("disclosure") or "PROTECTED").upper()
+        if disclosure not in VALID_DISCLOSURE_LEVELS:
+            logger.warning(
+                "devtorch parser: invalid disclosure level '%s' in sensitivity block — defaulting to PROTECTED | raw=%r",
+                disclosure, raw_tag,
+            )
+            disclosure = "PROTECTED"
+        sensitivities.append(
+            SensitivityBlock(
+                concept=concept,
+                signal=signal,
+                confidence=confidence,
+                disclosure=disclosure,
+            )
+        )
+
+    return commits, sensitivities
+
+
+def strip_racp_blocks(text: str) -> str:
+    """
+    Remove all ``<devtorch:commit .../>``, ``<devtorch:sensitivity .../>``,
+    and any ``<devtorch:*>...</devtorch:*>`` paired tags from *text*.
+
+    Returns clean text suitable for presentation to the user/caller.
+    Surrounding whitespace is collapsed but the overall structure is preserved.
+    """
+    # Remove paired tags first (they span more characters).
+    cleaned = _RE_PAIRED.sub("", text)
+    # Remove self-closing tags.
+    cleaned = _RE_SELF_CLOSING.sub("", cleaned)
+    # Collapse runs of blank lines that may have been left behind.
+    cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
+    return cleaned.strip()