crprotocol 2.0.0__py3-none-any.whl

crp/extraction/structured_output.py

@@ -0,0 +1,219 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Structured output handling — schema/grammar enforcement (§06 §6.9, 2J).
+
+Supports: Outlines FSM, GBNF grammar, logit masking, fallback JSON repair.
+All integrations are optional — graceful fallback if libraries unavailable.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# JSON repair (always available — no external deps)
+# ---------------------------------------------------------------------------
+
+def repair_json(raw: str) -> str | None:
+    """Best-effort repair of malformed JSON.
+
+    Handles: trailing commas, unquoted keys, single quotes, truncated output.
+    Returns the repaired JSON string, or None if unrecoverable.
+    """
+    # Strip markdown fences
+    cleaned = re.sub(r"```(?:json)?\s*", "", raw).strip()
+    cleaned = re.sub(r"```\s*$", "", cleaned).strip()
+
+    # Try direct parse first
+    try:
+        json.loads(cleaned)
+        return cleaned
+    except (json.JSONDecodeError, ValueError):
+        pass
+
+    # Fix single quotes → double quotes (naive — handles simple cases)
+    attempt = cleaned.replace("'", '"')
+    try:
+        json.loads(attempt)
+        return attempt
+    except (json.JSONDecodeError, ValueError):
+        pass
+
+    # Remove trailing commas before } or ]
+    attempt = re.sub(r",\s*([}\]])", r"\1", attempt)
+    try:
+        json.loads(attempt)
+        return attempt
+    except (json.JSONDecodeError, ValueError):
+        pass
+
+    # Try closing truncated output
+    open_braces = attempt.count("{") - attempt.count("}")
+    open_brackets = attempt.count("[") - attempt.count("]")
+    if open_braces > 0 or open_brackets > 0:
+        attempt += "}" * max(open_braces, 0)
+        attempt += "]" * max(open_brackets, 0)
+        try:
+            json.loads(attempt)
+            return attempt
+        except (json.JSONDecodeError, ValueError):
+            pass
+
+    return None
+
+
+def validate_json_schema(data: Any, schema: dict[str, Any]) -> list[str]:
+    """Validate *data* against JSON Schema. Returns list of error messages."""
+    try:
+        import jsonschema  # type: ignore[import-untyped]
+
+        validator = jsonschema.Draft7Validator(schema)
+        return [e.message for e in validator.iter_errors(data)]
+    except ImportError:
+        # jsonschema not installed — skip validation
+        return []
+
+
+# ---------------------------------------------------------------------------
+# Outlines FSM integration (optional)
+# ---------------------------------------------------------------------------
+
+class OutlinesFSMHandler:
+    """Outlines-based constrained generation via finite state machine."""
+
+    def __init__(self) -> None:
+        self._available: bool | None = None
+
+    @property
+    def is_available(self) -> bool:
+        if self._available is None:
+            try:
+                import outlines  # type: ignore[import-untyped]  # noqa: F401
+
+                self._available = True
+            except ImportError:
+                self._available = False
+        return self._available
+
+    def build_guide(self, schema: dict[str, Any]) -> Any:
+        """Build an Outlines JSON guide from a JSON Schema."""
+        if not self.is_available:
+            return None
+        try:
+            from outlines.generate import json as outlines_json  # type: ignore[import-untyped]
+
+            return outlines_json(schema)
+        except Exception:
+            logger.warning("Failed to build Outlines guide")
+            return None
+
+
+# ---------------------------------------------------------------------------
+# GBNF grammar support (for llama.cpp providers)
+# ---------------------------------------------------------------------------
+
+def json_schema_to_gbnf(schema: dict[str, Any]) -> str | None:
+    """Convert a simple JSON Schema to GBNF grammar string.
+
+    Handles flat object schemas with string/number/boolean/array properties.
+    Complex nested schemas require the full llama.cpp grammar converter.
+    """
+    props = schema.get("properties", {})
+    if not props:
+        return None
+
+    rules: list[str] = ['root ::= "{" ws']
+    prop_rules: list[str] = []
+
+    for i, (name, prop_schema) in enumerate(props.items()):
+        ptype = prop_schema.get("type", "string")
+        sep = ', "' if i > 0 else '"'
+        type_rule = _type_to_gbnf(ptype, name)
+        prop_rules.append(f'{sep}{name}": ' + type_rule)
+
+    rules.append(" ".join(prop_rules))
+    rules.append('ws "}"')
+    rules.append('ws ::= [ \\t\\n]*')
+    rules.append('string ::= "\\"" [^"\\\\]* "\\""')
+    rules.append('number ::= "-"? [0-9]+ ("." [0-9]+)?')
+    rules.append('boolean ::= "true" | "false"')
+
+    return "\n".join(rules)
+
+
+def _type_to_gbnf(json_type: str, _name: str) -> str:
+    mapping = {
+        "string": "ws string",
+        "number": "ws number",
+        "integer": "ws number",
+        "boolean": "ws boolean",
+    }
+    return mapping.get(json_type, "ws string")
+
+
+# ---------------------------------------------------------------------------
+# Composite handler
+# ---------------------------------------------------------------------------
+
+class StructuredOutputHandler:
+    """Orchestrates structured-output enforcement.
+
+    Priority order:
+    1. Outlines FSM (if available and provider supports it)
+    2. GBNF grammar (if provider is llama.cpp compatible)
+    3. Logit masking (if provider supports token-level constraints)
+    4. Fallback: post-hoc JSON repair + validation
+    """
+
+    def __init__(self) -> None:
+        self._outlines = OutlinesFSMHandler()
+
+    @property
+    def outlines_available(self) -> bool:
+        return self._outlines.is_available
+
+    def enforce(
+        self,
+        raw_output: str,
+        schema: dict[str, Any] | None = None,
+    ) -> tuple[Any | None, list[str]]:
+        """Attempt to parse and validate *raw_output* against *schema*.
+
+        Returns ``(parsed_data, errors)`` where *errors* is empty on success.
+        """
+        if schema is None:
+            # No schema — just try to parse as JSON
+            try:
+                return json.loads(raw_output), []
+            except (json.JSONDecodeError, ValueError):
+                repaired = repair_json(raw_output)
+                if repaired is not None:
+                    return json.loads(repaired), ["json_repaired"]
+                return None, ["json_parse_failed"]
+
+        # Try direct parse
+        try:
+            data = json.loads(raw_output)
+        except (json.JSONDecodeError, ValueError):
+            repaired = repair_json(raw_output)
+            if repaired is None:
+                return None, ["json_parse_failed"]
+            data = json.loads(repaired)
+
+        # Validate against schema
+        errors = validate_json_schema(data, schema)
+        return data, errors
+
+    def build_gbnf(self, schema: dict[str, Any]) -> str | None:
+        """Build a GBNF grammar string for llama.cpp providers."""
+        return json_schema_to_gbnf(schema)
+
+    def build_outlines_guide(self, schema: dict[str, Any]) -> Any:
+        """Build an Outlines FSM guide (returns None if unavailable)."""
+        return self._outlines.build_guide(schema)

crp/extraction/types.py

@@ -0,0 +1,299 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Extraction pipeline data types — Fact, FactEdge, FactGraph, ExtractionResult."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class ContentType(str, Enum):
+    """Content complexity classification for pipeline routing."""
+
+    ENTITY_RICH = "ENTITY_RICH"
+    REASONING_DENSE = "REASONING_DENSE"
+    NARRATIVE = "NARRATIVE"
+
+
+class RelationType(str, Enum):
+    """Semantic relation types for FactEdge records."""
+
+    CONDITION_FOR = "CONDITION_FOR"
+    CAUSE_EFFECT = "CAUSE_EFFECT"
+    CONTRAST = "CONTRAST"
+    CONCESSION = "CONCESSION"
+    CONSEQUENCE = "CONSEQUENCE"
+    ELABORATION = "ELABORATION"
+    SEQUENCE = "SEQUENCE"
+    RELATED = "RELATED"
+
+
+@dataclass
+class Fact:
+    """Single extracted fact from the extraction pipeline.
+
+    Lightweight record — embedding is computed lazily in Phase 4 (state layer).
+    """
+
+    # Metadata size limits (§audit M4)
+    MAX_METADATA_KEYS: int = 64
+    MAX_KEY_LENGTH: int = 128
+    MAX_VALUE_SIZE: int = 4096
+
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    text: str = ""
+    category: str = ""
+    source_window_id: str = ""
+    confidence: float = 0.0
+    extraction_stage: int = 0
+    created_at: float = field(default_factory=time.time)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    # Quality gate flags (set by post-extraction validation)
+    flagged_confidence: bool = False
+    confidence_flag_reason: str = ""
+
+    # Supersession (set by contradiction detection)
+    superseded_by: str | None = None
+    supersession_confidence: float = 0.0
+
+    def validate_metadata(self) -> None:
+        """Enforce metadata size limits (§audit M4).
+
+        Raises ValueError if metadata exceeds configured bounds.
+        """
+        if len(self.metadata) > self.MAX_METADATA_KEYS:
+            raise ValueError(
+                f"Fact metadata exceeds {self.MAX_METADATA_KEYS} keys "
+                f"(got {len(self.metadata)})"
+            )
+        for key, value in self.metadata.items():
+            if len(str(key)) > self.MAX_KEY_LENGTH:
+                raise ValueError(
+                    f"Metadata key exceeds {self.MAX_KEY_LENGTH} chars: {str(key)[:50]}..."
+                )
+            val_str = str(value)
+            if len(val_str) > self.MAX_VALUE_SIZE:
+                raise ValueError(
+                    f"Metadata value for '{key}' exceeds {self.MAX_VALUE_SIZE} chars "
+                    f"(got {len(val_str)})"
+                )
+
+    def set_metadata(self, key: str, value: Any) -> None:
+        """Set a metadata key with size validation."""
+        if len(str(key)) > self.MAX_KEY_LENGTH:
+            raise ValueError(f"Metadata key exceeds {self.MAX_KEY_LENGTH} chars")
+        if len(str(value)) > self.MAX_VALUE_SIZE:
+            raise ValueError(f"Metadata value exceeds {self.MAX_VALUE_SIZE} chars")
+        if key not in self.metadata and len(self.metadata) >= self.MAX_METADATA_KEYS:
+            raise ValueError(f"Metadata exceeds {self.MAX_METADATA_KEYS} keys limit")
+        self.metadata[key] = value
+
+
+@dataclass
+class FactEdge:
+    """Directed relation between two facts or text spans."""
+
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    source_id: str = ""
+    target_id: str = ""
+    relation_type: RelationType | str = RelationType.RELATED
+    confidence: float = 0.0
+    source_stage: int = 0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class FactGraph:
+    """In-memory graph of facts and edges."""
+
+    nodes: dict[str, Fact] = field(default_factory=dict)
+    edges: list[FactEdge] = field(default_factory=list)
+    # Edge indices for O(1) lookup (§audit L4)
+    _edges_from: dict[str, list[FactEdge]] = field(default_factory=lambda: {})
+    _edges_to: dict[str, list[FactEdge]] = field(default_factory=lambda: {})
+
+    def add_fact(self, fact: Fact) -> None:
+        self.nodes[fact.id] = fact
+
+    def remove_fact(self, fact_id: str) -> None:
+        """Remove a fact and all its edges from the graph (§audit2 STATE-H5)."""
+        self.nodes.pop(fact_id, None)
+        # Remove edges referencing this fact
+        self.edges = [e for e in self.edges
+                      if e.source_id != fact_id and e.target_id != fact_id]
+        # Clean edge indices
+        self._edges_from.pop(fact_id, None)
+        self._edges_to.pop(fact_id, None)
+        # Remove from other nodes' index entries
+        for idx in (self._edges_from, self._edges_to):
+            for key in list(idx):
+                idx[key] = [e for e in idx[key]
+                            if e.source_id != fact_id and e.target_id != fact_id]
+                if not idx[key]:
+                    del idx[key]
+
+    def add_edge(self, edge: FactEdge) -> None:
+        # Skip edges referencing non-existent facts (§audit G7)
+        if edge.source_id not in self.nodes or edge.target_id not in self.nodes:
+            return
+        self.edges.append(edge)
+        # Maintain O(1) edge indices (§audit L4)
+        self._edges_from.setdefault(edge.source_id, []).append(edge)
+        self._edges_to.setdefault(edge.target_id, []).append(edge)
+
+    def edges_from(self, fact_id: str) -> list[FactEdge]:
+        return list(self._edges_from.get(fact_id, []))
+
+    def edges_to(self, fact_id: str) -> list[FactEdge]:
+        return list(self._edges_to.get(fact_id, []))
+
+    def subgraph_for(self, fact_ids: set[str], max_hops: int = 1) -> FactGraph:
+        """Return subgraph containing *fact_ids* plus neighbours within *max_hops*."""
+        visited: set[str] = set(fact_ids)
+        frontier = set(fact_ids)
+        for _ in range(max_hops):
+            next_frontier: set[str] = set()
+            for fid in frontier:
+                for e in self._edges_from.get(fid, []):
+                    if e.target_id not in visited:
+                        next_frontier.add(e.target_id)
+                for e in self._edges_to.get(fid, []):
+                    if e.source_id not in visited:
+                        next_frontier.add(e.source_id)
+            visited |= next_frontier
+            frontier = next_frontier
+        sub_nodes = {fid: self.nodes[fid] for fid in visited if fid in self.nodes}
+        sub_edges = [e for e in self.edges if e.source_id in visited and e.target_id in visited]
+        sub = FactGraph(nodes=sub_nodes, edges=sub_edges)
+        # Rebuild edge indices for the subgraph (§audit2 STATE-H4)
+        for e in sub_edges:
+            sub._edges_from.setdefault(e.source_id, []).append(e)
+            sub._edges_to.setdefault(e.target_id, []).append(e)
+        return sub
+
+    def serialize_for_envelope(self) -> str:
+        """Plain-text serialisation for envelope packing."""
+        lines: list[str] = []
+        for fid, fact in self.nodes.items():
+            lines.append(f"- {fact.text}")
+            for edge in self.edges_from(fid):
+                target = self.nodes.get(edge.target_id)
+                if target:
+                    rel = edge.relation_type
+                    if isinstance(rel, RelationType):
+                        rel = rel.value
+                    lines.append(f"  ↳ [{rel}] {target.text}")
+        return "\n".join(lines)
+
+
+class ValidationSeverity(str, Enum):
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+
+
+@dataclass
+class ValidationIssue:
+    """Single issue found by the quality gate."""
+
+    type: str = ""
+    severity: ValidationSeverity = ValidationSeverity.LOW
+    detail: str = ""
+
+
+@dataclass
+class ValidationResult:
+    """Result from one quality-gate tier."""
+
+    tier: int = 0
+    passed: bool = True
+    issues: list[ValidationIssue] = field(default_factory=list)
+
+
+@dataclass
+class Contradiction:
+    """A detected contradiction between two facts."""
+
+    fact_a: Fact | None = None
+    fact_b: Fact | None = None
+    similarity: float = 0.0
+    content_diff: float = 0.0
+    confidence: float = 0.0
+
+
+@dataclass
+class FactEvent:
+    """Immutable audit-log entry for fact lifecycle events."""
+
+    event_id: int = 0
+    timestamp: float = field(default_factory=time.time)
+    window_id: str = ""
+    event_type: str = ""  # "created" | "superseded" | "compacted" | "archived" | "restored"
+    fact_id: str = ""
+    payload: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class ExtractionResult:
+    """Complete extraction result from the graduated pipeline."""
+
+    extraction_id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    source_window_id: str = ""
+    timestamp: float = field(default_factory=time.time)
+
+    # Extracted data
+    facts: list[Fact] = field(default_factory=list)
+    edges: list[FactEdge] = field(default_factory=list)
+    fact_graph: FactGraph = field(default_factory=FactGraph)
+
+    # Pipeline execution
+    stages_run: list[int] = field(default_factory=list)
+    stages_skipped: list[int] = field(default_factory=list)
+    total_extraction_latency_ms: float = 0.0
+    per_stage_latency: dict[int, float] = field(default_factory=dict)
+
+    # Quality metrics
+    total_facts: int = 0
+    total_edges: int = 0
+    average_confidence: float = 0.0
+    entity_density: float = 0.0
+    relation_density: float = 0.0
+
+    # Content classification
+    content_type: ContentType = ContentType.NARRATIVE
+    discourse_markers_found: int = 0
+
+    # Pipeline state (for self-calibration)
+    stage_yields: dict[int, int] = field(default_factory=dict)
+    escalation_triggers: list[str] = field(default_factory=list)
+
+    # Quality gate
+    quality_gate_passed: bool = True
+    quality_issues: list[str] = field(default_factory=list)
+
+    # Normalization
+    facts_after_normalization: int = 0
+
+    @property
+    def success(self) -> bool:
+        return self.quality_gate_passed
+
+    def finalize(self) -> None:
+        """Compute aggregate metrics from facts/edges lists."""
+        self.total_facts = len(self.facts)
+        self.total_edges = len(self.edges)
+        if self.facts:
+            self.average_confidence = sum(f.confidence for f in self.facts) / len(self.facts)
+        self.relation_density = self.total_edges / max(self.total_facts, 1)
+        self.facts_after_normalization = self.total_facts
+        # Build graph
+        self.fact_graph = FactGraph()
+        for f in self.facts:
+            self.fact_graph.add_fact(f)
+        for e in self.edges:
+            self.fact_graph.add_edge(e)