crprotocol 2.0.0__py3-none-any.whl

crp/observability/quality.py

@@ -0,0 +1,135 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Quality tier classification and reporting (§05, §10).
+
+Each CRP session runs at a quality tier that tells the user how well
+the context pipeline is performing:
+
+    S  — all signals green, overhead < 5 %
+    A  — minor gaps, overhead < 10 %
+    B  — acceptable, overhead < 15 %
+    C  — degraded, hierarchical processing needed
+    D  — minimal / fallback mode
+
+QualityReporter takes a handful of easily-computed metrics and maps
+them to one of these tiers.  It can also detect *degradation* (tier
+dropping below a previous high-water mark).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+
+class QualityTier(Enum):
+    """CRP quality tiers (best → worst)."""
+
+    S = "S"
+    A = "A"
+    B = "B"
+    C = "C"
+    D = "D"
+
+    def __lt__(self, other: QualityTier) -> bool:
+        order = ["S", "A", "B", "C", "D"]
+        return order.index(self.value) < order.index(other.value)
+
+
+# Threshold rules: (max_overhead_pct, max_fact_miss_pct, tier)
+# Checked in order — first match wins.
+_TIER_RULES: list[tuple[float, float, QualityTier]] = [
+    (5.0, 2.0, QualityTier.S),
+    (10.0, 10.0, QualityTier.A),
+    (15.0, 25.0, QualityTier.B),
+    (30.0, 50.0, QualityTier.C),
+    # Everything else → D
+]
+
+
+@dataclass
+class QualityReport:
+    """Result of a quality assessment."""
+
+    tier: QualityTier
+    overhead_pct: float
+    fact_miss_pct: float
+    degraded: bool = False  # True if tier dropped below previous high
+    details: dict[str, Any] | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "tier": self.tier.value,
+            "overhead_pct": round(self.overhead_pct, 2),
+            "fact_miss_pct": round(self.fact_miss_pct, 2),
+            "degraded": self.degraded,
+            "details": self.details or {},
+        }
+
+
+class QualityReporter:
+    """Classify and track quality tiers across a session.
+
+    Usage::
+
+        qr = QualityReporter()
+        report = qr.assess(overhead_pct=4.2, fact_miss_pct=1.0)
+        assert report.tier == QualityTier.S
+
+        # Later, if overhead rises:
+        report2 = qr.assess(overhead_pct=18.0, fact_miss_pct=30.0)
+        assert report2.tier == QualityTier.C
+        assert report2.degraded  # dropped from S → C
+    """
+
+    def __init__(self) -> None:
+        self._high_water: QualityTier | None = None
+        self._history: list[QualityReport] = []
+
+    def assess(
+        self,
+        overhead_pct: float,
+        fact_miss_pct: float,
+        details: dict[str, Any] | None = None,
+    ) -> QualityReport:
+        """Compute the current quality tier and check for degradation.
+
+        Args:
+            overhead_pct:  Current overhead as a percentage (0-100).
+            fact_miss_pct: Percentage of available facts *not* included
+                           in the envelope (0-100).
+            details:       Optional opaque dict attached to the report.
+        """
+        tier = self._classify(overhead_pct, fact_miss_pct)
+        degraded = self._high_water is not None and self._high_water < tier
+
+        # Update high-water mark.
+        if self._high_water is None or tier < self._high_water:
+            self._high_water = tier
+
+        report = QualityReport(
+            tier=tier,
+            overhead_pct=overhead_pct,
+            fact_miss_pct=fact_miss_pct,
+            degraded=degraded,
+            details=details,
+        )
+        self._history.append(report)
+        return report
+
+    @property
+    def current_tier(self) -> QualityTier | None:
+        """Most recently assessed tier, or ``None`` if never assessed."""
+        return self._history[-1].tier if self._history else None
+
+    @property
+    def history(self) -> list[QualityReport]:
+        return list(self._history)
+
+    @staticmethod
+    def _classify(overhead_pct: float, fact_miss_pct: float) -> QualityTier:
+        for max_oh, max_fm, tier in _TIER_RULES:
+            if overhead_pct <= max_oh and fact_miss_pct <= max_fm:
+                return tier
+        return QualityTier.D

crp/observability/structured_logging.py

@@ -0,0 +1,81 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Structured logging with correlation IDs for CRP pipeline (§audit H9).
+
+Provides:
+- Per-request correlation IDs propagated across dispatch pipeline stages
+- JSON-structured log formatter for machine-parseable logs
+- Context-local storage for request metadata
+"""
+
+from __future__ import annotations
+
+import contextvars
+import json
+import logging
+import time
+import uuid
+from typing import Any
+
+# Context variable for correlation ID — propagated across async boundaries
+_correlation_id: contextvars.ContextVar[str] = contextvars.ContextVar(
+    "crp_correlation_id", default=""
+)
+_session_id: contextvars.ContextVar[str] = contextvars.ContextVar(
+    "crp_session_id", default=""
+)
+
+
+def new_correlation_id() -> str:
+    """Generate and set a new correlation ID for the current context."""
+    cid = uuid.uuid4().hex[:12]
+    _correlation_id.set(cid)
+    return cid
+
+
+def get_correlation_id() -> str:
+    """Get the current correlation ID (empty string if none set)."""
+    return _correlation_id.get()
+
+
+def set_session_context(session_id: str) -> None:
+    """Set the session ID for the current logging context."""
+    _session_id.set(session_id)
+
+
+class StructuredFormatter(logging.Formatter):
+    """JSON log formatter with automatic correlation ID injection."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        entry: dict[str, Any] = {
+            "ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)),
+            "level": record.levelname,
+            "logger": record.name,
+            "msg": record.getMessage(),
+        }
+
+        cid = _correlation_id.get()
+        if cid:
+            entry["correlation_id"] = cid
+
+        sid = _session_id.get()
+        if sid:
+            entry["session_id"] = sid
+
+        if record.exc_info and record.exc_info[1]:
+            entry["exception"] = str(record.exc_info[1])
+
+        return json.dumps(entry, separators=(",", ":"))
+
+
+def configure_structured_logging(level: int = logging.INFO) -> None:
+    """Configure the root 'crp' logger with structured JSON output.
+
+    Call once at application startup for machine-parseable logs.
+    """
+    root_logger = logging.getLogger("crp")
+    if not root_logger.handlers:
+        handler = logging.StreamHandler()
+        handler.setFormatter(StructuredFormatter())
+        root_logger.addHandler(handler)
+    root_logger.setLevel(level)

crp/observability/telemetry.py

@@ -0,0 +1,117 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Per-window telemetry writer — ``telemetry.jsonl`` output (§8.9).
+
+TelemetryWriter appends one JSON object per line for every completed
+window.  The JSONL file can be consumed by downstream analytics,
+dashboards, or the AuditLog.
+
+Design goals:
+  * One line per window → easy to ``grep``, ``jq``, or stream-parse.
+  * Flush after every write so no data is lost on crash.
+  * Works with file paths or any writable file-like object.
+"""
+
+from __future__ import annotations
+
+import atexit
+import json
+import os
+import time
+from dataclasses import asdict, dataclass, field
+from typing import IO, Any
+
+
+@dataclass
+class WindowTelemetry:
+    """Telemetry record for a single completed window.
+
+    Fields mirror the most useful per-window stats.  Extra fields can
+    be added via ``extra``.
+    """
+
+    session_id: str
+    window_id: str
+    timestamp: float = field(default_factory=time.time)
+    input_tokens: int = 0
+    output_tokens: int = 0
+    overhead_tokens: int = 0
+    facts_included: int = 0
+    facts_available: int = 0
+    latency_ms: float = 0.0
+    quality_tier: str = ""
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        d = asdict(self)
+        # Flatten extra into the top level for ease of querying.
+        extra = d.pop("extra", {})
+        d.update(extra)
+        return d
+
+
+class TelemetryWriter:
+    """Append-only JSONL writer for per-window telemetry.
+
+    Usage::
+
+        tw = TelemetryWriter("telemetry.jsonl")
+        tw.write(WindowTelemetry(session_id="abc", window_id="w0", ...))
+        tw.close()
+
+    Also works as a context manager::
+
+        with TelemetryWriter("telemetry.jsonl") as tw:
+            tw.write(record)
+    """
+
+    def __init__(self, path_or_file: str | IO[str]) -> None:
+        if isinstance(path_or_file, str):
+            # Ensure parent directory exists.
+            parent = os.path.dirname(path_or_file)
+            if parent:
+                os.makedirs(parent, exist_ok=True)
+            self._file: IO[str] = open(path_or_file, "a", encoding="utf-8")  # noqa: SIM115
+            self._owns_file = True
+        else:
+            self._file = path_or_file
+            self._owns_file = False
+        self._count = 0
+        self._closed = False
+        # Ensure close on interpreter shutdown (§audit H3)
+        atexit.register(self.close)
+
+    # -- writing ----------------------------------------------------------
+
+    def write(self, record: WindowTelemetry) -> None:
+        """Serialize *record* and append one JSON line. Flushes immediately."""
+        line = json.dumps(record.to_dict(), separators=(",", ":"))
+        self._file.write(line + "\n")
+        self._file.flush()
+        self._count += 1
+
+    @property
+    def records_written(self) -> int:
+        return self._count
+
+    # -- lifecycle --------------------------------------------------------
+
+    def close(self) -> None:
+        if self._closed:
+            return
+        self._closed = True
+        if self._owns_file:
+            self._file.close()
+
+    def __del__(self) -> None:
+        """Safety net: close file if not already closed (§audit H3)."""
+        try:
+            self.close()
+        except Exception:  # noqa: BLE001
+            pass
+
+    def __enter__(self) -> TelemetryWriter:
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()

crp/provenance/__init__.py

@@ -0,0 +1,314 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Decision Provenance Engine (DPE) — §7.14.3.
+
+Orchestrates claim detection, attribution scoring, provenance chain
+construction, and report generation for every LLM dispatch window.
+
+Usage::
+
+    from crp.provenance import DecisionProvenanceEngine, ProvenanceConfig
+
+    dpe = DecisionProvenanceEngine(config=ProvenanceConfig())
+    report = dpe.analyse(
+        output_text="The server uses AES-256 encryption...",
+        packed_facts=envelope_result.packed_facts,
+        session_id=session.id,
+        window_id=window.id,
+    )
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Sequence
+
+from crp.envelope.packer import PackedFact
+
+from ._types import (
+    AttributionType,
+    ClaimAttribution,
+    ClaimType,
+    EntailmentResult,
+    FactScore,
+    FidelityReport,
+    HallucinationRiskReport,
+    OmissionSeverity,
+    ProvenanceChain,
+    ProvenanceConfig,
+    ProvenanceLink,
+    ProvenanceReport,
+)
+from .attribution_scorer import score_all_claims
+from .claim_detector import DetectedClaim, detect_claims
+from .contradiction_detector import detect_contradictions
+from .distortion_detector import detect_distortions
+from .entailment_verifier import verify_entailment
+from .fabrication_detector import detect_fabrications
+from .hallucination_scorer import score_hallucination_risk
+from .omission_analyzer import analyze_omissions
+from .provenance_chain import build_all_chains, enrich_fact_metadata
+from .report_generator import generate_json_report, generate_markdown_report
+
+
+__all__ = [
+    "DecisionProvenanceEngine",
+    "ProvenanceConfig",
+    "ProvenanceReport",
+    "FidelityReport",
+    "ClaimAttribution",
+    "ClaimType",
+    "AttributionType",
+    "FactScore",
+    "ProvenanceChain",
+    "ProvenanceLink",
+    "DetectedClaim",
+    "detect_claims",
+    "score_all_claims",
+    "build_all_chains",
+    "enrich_fact_metadata",
+    "generate_markdown_report",
+    "generate_json_report",
+    "detect_distortions",
+    "detect_fabrications",
+    "analyze_omissions",
+    "detect_contradictions",
+    "verify_entailment",
+    "score_hallucination_risk",
+    "EntailmentResult",
+    "HallucinationRiskReport",
+]
+
+
+class DecisionProvenanceEngine:
+    """Main entry point for the Decision Provenance Engine.
+
+    Ties together claim detection → attribution scoring → provenance chain
+    construction → report generation into a single ``analyse()`` call.
+    """
+
+    def __init__(self, *, config: ProvenanceConfig | None = None) -> None:
+        self._config = config or ProvenanceConfig()
+
+    # -- Properties ---------------------------------------------------------
+
+    @property
+    def config(self) -> ProvenanceConfig:
+        return self._config
+
+    @property
+    def enabled(self) -> bool:
+        return self._config.enabled
+
+    # -- Main pipeline ------------------------------------------------------
+
+    def analyse(
+        self,
+        output_text: str,
+        packed_facts: Sequence[PackedFact],
+        *,
+        session_id: str = "",
+        window_id: str = "",
+        envelope_saturation: float = 0.0,
+        task_input_preview: str = "",
+        fact_metadata: dict[str, dict[str, object]] | None = None,
+    ) -> ProvenanceReport:
+        """Run the full DPE pipeline and return a provenance report.
+
+        Args:
+            output_text: Raw LLM output text for this window.
+            packed_facts: Facts that were packed into the envelope.
+            session_id: Current CRP session ID.
+            window_id: Current dispatch window ID.
+            envelope_saturation: Envelope token saturation ratio (0.0-1.0).
+            task_input_preview: First 120 chars of the task input.
+            fact_metadata: Optional mapping of fact_id → metadata dict
+                           containing ``source_window_id`` and
+                           ``extraction_stage`` for chain enrichment.
+
+        Returns:
+            ProvenanceReport with attribution data, chains, and counts.
+        """
+        if not self._config.enabled:
+            return ProvenanceReport(
+                session_id=session_id,
+                window_id=window_id,
+                timestamp=time.time(),
+            )
+
+        facts_list = list(packed_facts)
+
+        # Step 1: Detect claims
+        claims = detect_claims(
+            output_text,
+            min_length=self._config.min_claim_length,
+            max_claims=self._config.max_claims_per_output,
+        )
+
+        # Step 2: Score claims against facts
+        attributions = score_all_claims(claims, facts_list, config=self._config)
+
+        # Step 3: Enrich with fact metadata (if available)
+        if fact_metadata:
+            enrich_fact_metadata(attributions, fact_metadata)
+
+        # Step 4: Build provenance chains
+        chains = build_all_chains(
+            attributions,
+            session_id=session_id,
+            window_id=window_id,
+            envelope_saturation=envelope_saturation,
+            envelope_facts_included=len(facts_list),
+            task_input_preview=task_input_preview,
+        )
+
+        # Step 5: Fidelity verification
+        distortions = detect_distortions(attributions, facts_list)
+        omissions = analyze_omissions(attributions, facts_list)
+        fabrications = detect_fabrications(attributions, facts_list)
+        contradictions = detect_contradictions(attributions)
+
+        critical_omissions = sum(
+            1 for o in omissions if o.severity == OmissionSeverity.CRITICAL
+        )
+
+        # Composite fidelity score: start at 1.0, deduct for issues
+        # Normalize penalties by total claims to avoid score collapse on
+        # long outputs.  With 30 claims, 3 distortions should not
+        # obliterate the score the same way it would for 3 claims.
+        _total = max(len(attributions), 1)
+        fidelity_score = 1.0
+        fidelity_score -= (len(distortions) / _total) * 0.40
+        fidelity_score -= (len(fabrications) / _total) * 0.35
+        fidelity_score -= (critical_omissions / _total) * 0.25
+        fidelity_score -= (len(contradictions) / _total) * 0.50
+        fidelity_score = max(0.0, round(fidelity_score, 4))
+
+        fidelity = FidelityReport(
+            distortions=distortions,
+            fabrications=fabrications,
+            omissions=omissions,
+            contradictions=contradictions,
+            distortion_count=len(distortions),
+            fabrication_count=len(fabrications),
+            critical_omission_count=critical_omissions,
+            contradiction_count=len(contradictions),
+            fidelity_score=fidelity_score,
+        )
+
+        # Step 6: Semantic entailment verification
+        entailment_results: list[EntailmentResult] = []
+        if self._config.entailment_enabled:
+            entailment_results = verify_entailment(
+                attributions, facts_list, config=self._config,
+            )
+
+        # Step 7: Hallucination risk scoring
+        risk_report: HallucinationRiskReport | None = None
+        if self._config.risk_scoring_enabled:
+            risk_report = score_hallucination_risk(
+                attributions,
+                fidelity=fidelity,
+                entailment_results=entailment_results,
+                config=self._config,
+            )
+
+        # Step 8: Compute counts and build report
+        report = self._build_report(
+            claims=claims,
+            attributions=attributions,
+            chains=chains,
+            session_id=session_id,
+            window_id=window_id,
+            output_text=output_text,
+            facts_count=len(facts_list),
+            fidelity=fidelity,
+            entailment_results=entailment_results,
+            risk_report=risk_report,
+        )
+
+        return report
+
+    # -- Report assembly ----------------------------------------------------
+
+    def _build_report(
+        self,
+        *,
+        claims: list[DetectedClaim],
+        attributions: list[ClaimAttribution],
+        chains: list[ProvenanceChain],
+        session_id: str,
+        window_id: str,
+        output_text: str,
+        facts_count: int,
+        fidelity: FidelityReport | None = None,
+        entailment_results: list[EntailmentResult] | None = None,
+        risk_report: HallucinationRiskReport | None = None,
+    ) -> ProvenanceReport:
+        """Assemble counts and build the ProvenanceReport."""
+        # Claim type counts
+        factual = sum(1 for c in claims if c.claim_type == ClaimType.FACTUAL_CLAIM)
+        opinion = sum(1 for c in claims if c.claim_type == ClaimType.OPINION)
+        procedural = sum(1 for c in claims if c.claim_type == ClaimType.PROCEDURAL)
+        hedge = sum(1 for c in claims if c.claim_type == ClaimType.HEDGE)
+        connective = sum(1 for c in claims if c.claim_type == ClaimType.CONNECTIVE)
+
+        # Attribution type counts
+        grounded = sum(
+            1 for a in attributions
+            if a.attribution_type == AttributionType.CONTEXT_GROUNDED
+        )
+        parametric = sum(
+            1 for a in attributions
+            if a.attribution_type == AttributionType.PARAMETRIC
+        )
+        mixed = sum(
+            1 for a in attributions
+            if a.attribution_type == AttributionType.MIXED
+        )
+        uncertain = sum(
+            1 for a in attributions
+            if a.attribution_type == AttributionType.UNCERTAIN
+        )
+
+        # Grounding ratio: context-grounded factual claims / total factual claims
+        scorable = sum(
+            1 for a in attributions
+            if a.claim_type in (ClaimType.FACTUAL_CLAIM, ClaimType.HEDGE)
+        )
+        grounding_ratio = grounded / scorable if scorable > 0 else 0.0
+
+        return ProvenanceReport(
+            session_id=session_id,
+            window_id=window_id,
+            timestamp=time.time(),
+            total_claims=len(claims),
+            factual_claims=factual,
+            opinion_claims=opinion,
+            procedural_claims=procedural,
+            hedge_claims=hedge,
+            connective_claims=connective,
+            context_grounded_count=grounded,
+            parametric_count=parametric,
+            mixed_count=mixed,
+            uncertain_count=uncertain,
+            grounding_ratio=round(grounding_ratio, 4),
+            attributions=attributions,
+            chains=chains,
+            chain_verified=True,
+            output_token_count=len(output_text.split()),
+            envelope_facts_count=facts_count,
+            fidelity=fidelity,
+            entailment_results=entailment_results or [],
+            risk_report=risk_report,
+        )
+
+    # -- Report formatting helpers ------------------------------------------
+
+    def generate_markdown(self, report: ProvenanceReport) -> str:
+        """Generate a human-readable markdown provenance report."""
+        return generate_markdown_report(report)
+
+    def generate_json(self, report: ProvenanceReport) -> dict:
+        """Generate a machine-readable JSON provenance report."""
+        return generate_json_report(report)