crprotocol 2.0.0__py3-none-any.whl

crp/extraction/pipeline.py

@@ -0,0 +1,360 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Extraction pipeline orchestration — blackboard-reactive 6-stage pipeline.
+
+Implements the graduated extraction decision tree, self-calibrating baselines,
+and stage escalation logic per §3.2.
+"""
+
+from __future__ import annotations
+
+import logging
+import statistics
+import time
+from dataclasses import dataclass, field
+
+from crp.core.task_intent import TaskIntent
+from crp.extraction.complexity import detect_content_complexity
+from crp.extraction.stage1_regex import RegexExtractor
+from crp.extraction.stage2_statistical import StatisticalExtractor
+from crp.extraction.stage3_gliner import GLiNERExtractor, derive_labels_from_noun_phrases
+from crp.extraction.stage4_uie import UIEExtractor
+from crp.extraction.stage5_discourse import DiscourseExtractor, count_discourse_markers
+from crp.extraction.stage6_llm import DispatchFn, LLMExtractor
+from crp.extraction.types import (
+    ContentType,
+    ExtractionResult,
+    Fact,
+    FactEdge,
+)
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Self-calibrating baseline
+# ---------------------------------------------------------------------------
+
+_CALIBRATION_WINDOW_COUNT = 5   # Initial lock after N windows
+_RECALIBRATION_INTERVAL = 10   # Recalibrate every N windows after lock
+_ROLLING_WINDOW_SIZE = 10      # Rolling window for recalibration
+_DEFAULT_STAGE2_BASELINE = 10  # Default facts before calibration
+_DEFAULT_STAGE3_BASELINE = 5
+_DEFAULT_CONFIDENCE_FLOOR = 0.6
+_DRIFT_THRESHOLD = 0.30        # Recalibrate if yield drifts > 30% from baseline
+# Gap G fix: Raised threshold so ML stages (GLiNER, UIE) fire more often.
+# At 50, stages 1-2 would short-circuit on moderately rich text, preventing
+# NER and relation extraction.  At 120, ML stages run for most content,
+# producing richer fact graphs and CKF edges.
+_SHORT_CIRCUIT_THRESHOLD = 120  # Skip later stages only with very rich extraction (§audit M6)
+
+
+@dataclass
+class CalibrationState:
+    """Tracks self-calibrating baselines for stage escalation.
+
+    Baselines are initially locked after ``_CALIBRATION_WINDOW_COUNT``
+    windows.  After that, the system *periodically recalibrates* every
+    ``_RECALIBRATION_INTERVAL`` windows using a rolling window of the
+    most recent results.  This prevents stale baselines when extraction
+    profiles change (e.g. when new content domains appear or stages
+    come online).
+    """
+
+    baseline_stage_2: float = _DEFAULT_STAGE2_BASELINE
+    baseline_stage_3: float = _DEFAULT_STAGE3_BASELINE
+    baseline_confidence_floor: float = _DEFAULT_CONFIDENCE_FLOOR
+    baseline_locked: bool = False
+    calibration_epoch: int = 0  # How many times baselines have been calibrated
+    _results: list[ExtractionResult] = field(default_factory=list)
+
+    def record(self, result: ExtractionResult) -> None:
+        """Record an extraction result. Calibrates/recalibrates as needed."""
+        self._results.append(result)
+        n = len(self._results)
+
+        if not self.baseline_locked and n >= _CALIBRATION_WINDOW_COUNT:
+            # Initial calibration
+            self._calibrate(self._results[:_CALIBRATION_WINDOW_COUNT])
+        elif self.baseline_locked and n % _RECALIBRATION_INTERVAL == 0:
+            # Periodic recalibration with rolling window
+            recent = self._results[-_ROLLING_WINDOW_SIZE:]
+            if self._detect_drift(recent):
+                logger.info("Baseline drift detected at window %d — recalibrating", n)
+                self._calibrate(recent)
+
+        # Cap stored results to prevent unbounded growth
+        _MAX_RESULTS = _ROLLING_WINDOW_SIZE * 3
+        if len(self._results) > _MAX_RESULTS:
+            self._results = self._results[-_ROLLING_WINDOW_SIZE:]
+
+    def _calibrate(self, results: list[ExtractionResult]) -> None:
+        """Compute baselines from the given result set."""
+        # Stage 2 yield mean
+        s2 = [r.stage_yields.get(2, 0) for r in results]
+        if s2:
+            self.baseline_stage_2 = statistics.mean(s2)
+
+        # Stage 3 yield mean (if ran)
+        s3 = [r.stage_yields.get(3, 0) for r in results if 3 in r.stages_run]
+        if s3:
+            self.baseline_stage_3 = statistics.mean(s3)
+
+        # Confidence 10th percentile
+        all_conf = [f.confidence for r in results for f in r.facts]
+        if len(all_conf) >= 10:
+            self.baseline_confidence_floor = statistics.quantiles(all_conf, n=10)[0]
+
+        self.baseline_locked = True
+        self.calibration_epoch += 1
+        logger.info(
+            "Baselines calibrated (epoch %d) — stage2=%.1f, stage3=%.1f, conf_floor=%.2f",
+            self.calibration_epoch,
+            self.baseline_stage_2,
+            self.baseline_stage_3,
+            self.baseline_confidence_floor,
+        )
+
+    def _detect_drift(self, recent: list[ExtractionResult]) -> bool:
+        """Return True if recent yields have drifted significantly from baselines."""
+        if not recent:
+            return False
+        s2_recent = [r.stage_yields.get(2, 0) for r in recent]
+        if s2_recent and self.baseline_stage_2 > 0:
+            mean_s2 = statistics.mean(s2_recent)
+            drift = abs(mean_s2 - self.baseline_stage_2) / max(self.baseline_stage_2, 1)
+            if drift > _DRIFT_THRESHOLD:
+                return True
+        return False
+
+    def should_escalate_stage_3(self, stage_1_2_yield: int) -> bool:
+        return stage_1_2_yield < self.baseline_stage_2
+
+    def should_escalate_stage_4(self, stage_3_relation_yield: float) -> bool:
+        return stage_3_relation_yield < 0.1
+
+    @property
+    def results_count(self) -> int:
+        return len(self._results)
+
+
+# ---------------------------------------------------------------------------
+# Pipeline
+# ---------------------------------------------------------------------------
+
+class ExtractionPipeline:
+    """Blackboard-reactive 6-stage extraction pipeline.
+
+    Usage::
+
+        pipeline = ExtractionPipeline()
+        result = pipeline.extract(text, task_intent)
+    """
+
+    def __init__(
+        self,
+        *,
+        enable_stage_3: bool = True,
+        enable_stage_4: bool = True,
+        enable_stage_5: bool = True,
+        enable_stage_6: bool = False,  # MAY — disabled by default
+        dispatch_fn: DispatchFn | None = None,
+        short_circuit_threshold: int = _SHORT_CIRCUIT_THRESHOLD,
+    ) -> None:
+        self._stage1 = RegexExtractor()
+        self._stage2 = StatisticalExtractor()
+        self._stage3 = GLiNERExtractor()
+        self._stage4 = UIEExtractor()
+        self._stage5 = DiscourseExtractor()
+        self._stage6 = LLMExtractor(dispatch_fn)
+
+        self._enable_3 = enable_stage_3
+        self._enable_4 = enable_stage_4
+        self._enable_5 = enable_stage_5
+        self._enable_6 = enable_stage_6
+        self._short_circuit_threshold = short_circuit_threshold
+
+        self._calibration = CalibrationState()
+
+        # Eagerly load enabled ML models at init to avoid cold-start latency
+        # during first dispatch. GLiNER (~200MB) takes 10-20s to load; doing it
+        # here means the cost is paid once at protocol start, not mid-pipeline.
+        if self._enable_3:
+            self._stage3._ensure_model()
+        if self._enable_4:
+            self._stage4._ensure_model()
+
+    # -- Configuration ------------------------------------------------------
+
+    @property
+    def calibration(self) -> CalibrationState:
+        return self._calibration
+
+    def set_dispatch_fn(self, fn: DispatchFn) -> None:
+        self._stage6.set_dispatch(fn)
+
+    def register_regex_pattern(
+        self, name: str, pattern: str, category: str, confidence: float = 0.90,
+    ) -> None:
+        self._stage1.register_pattern(name, pattern, category, confidence)
+
+    # -- Extraction ---------------------------------------------------------
+
+    def extract(
+        self,
+        text: str,
+        task_intent: TaskIntent | None = None,
+        source_window_id: str = "",
+    ) -> ExtractionResult:
+        """Run the graduated extraction pipeline.
+
+        Stages 1-2 always run. Stages 3-6 run conditionally based on
+        content complexity, yield thresholds, and availability.
+        """
+        t0 = time.monotonic()
+        result = ExtractionResult(source_window_id=source_window_id)
+        all_facts: list[Fact] = []
+        all_edges: list[FactEdge] = []
+        stage_latency: dict[int, float] = {}
+
+        # -- Content complexity detection -----------------------------------
+        content_type = detect_content_complexity(text)
+        result.content_type = content_type
+        result.discourse_markers_found = count_discourse_markers(text)
+
+        word_count = max(len(text.split()), 1)
+
+        # ── STAGE 1: Regex (ALWAYS) ───────────────────────────────────────
+        ts = time.monotonic()
+        s1_facts = self._stage1.extract(text, source_window_id)
+        stage_latency[1] = (time.monotonic() - ts) * 1000
+        all_facts.extend(s1_facts)
+        result.stages_run.append(1)
+        result.stage_yields[1] = len(s1_facts)
+
+        # ── STAGE 2: Statistical (ALWAYS) ─────────────────────────────────
+        ts = time.monotonic()
+        s2_facts = self._stage2.extract(text, source_window_id)
+        stage_latency[2] = (time.monotonic() - ts) * 1000
+        all_facts.extend(s2_facts)
+        result.stages_run.append(2)
+        result.stage_yields[2] = len(s2_facts)
+
+        combined_s1_s2 = len(s1_facts) + len(s2_facts)
+
+        # Short-circuit: skip expensive ML stages if early stages extracted enough (§audit M6)
+        _short_circuited = (
+            self._short_circuit_threshold > 0
+            and combined_s1_s2 >= self._short_circuit_threshold
+        )
+        if _short_circuited:
+            logger.info(
+                "Short-circuit: %d facts from stages 1-2 >= threshold %d, skipping stages 3-6",
+                combined_s1_s2, self._short_circuit_threshold,
+            )
+            result.stages_skipped.extend([s for s in (3, 4, 5, 6) if s not in result.stages_skipped])
+
+        # ── STAGE 3: GLiNER (conditional) ─────────────────────────────────
+        if not _short_circuited and self._enable_3 and self._calibration.should_escalate_stage_3(combined_s1_s2):
+            from crp.license_guard import is_feature_allowed
+            if not is_feature_allowed("stage_3"):
+                result.stages_skipped.append(3)
+            elif self._stage3.is_available:
+                ts = time.monotonic()
+                # Derive labels from Stage 2 noun phrases
+                noun_phrases = [f.text for f in s2_facts if f.category == "noun_phrase"]
+                labels = derive_labels_from_noun_phrases(noun_phrases) if noun_phrases else None
+                s3_facts = self._stage3.extract(
+                    text, labels=labels, source_window_id=source_window_id,
+                )
+                stage_latency[3] = (time.monotonic() - ts) * 1000
+                all_facts.extend(s3_facts)
+                result.stages_run.append(3)
+                result.stage_yields[3] = len(s3_facts)
+                result.escalation_triggers.append(
+                    f"stage_1_2_yield={combined_s1_s2} < baseline={self._calibration.baseline_stage_2:.0f}"
+                )
+            else:
+                result.stages_skipped.append(3)
+        else:
+            result.stages_skipped.append(3)
+
+        # ── STAGE 4: UIE (conditional) ────────────────────────────────────
+        stage_3_ran = 3 in result.stages_run
+        if not _short_circuited and self._enable_4 and stage_3_ran:
+            # Relation yield from Stage 3 — approximate by counting facts / sentences
+            _sent_count = max(len(text.split(".")), 1)
+            s3_yield = result.stage_yields.get(3, 0)
+            relation_per_sent = s3_yield / _sent_count
+            if self._calibration.should_escalate_stage_4(relation_per_sent):
+                if self._stage4.is_available:
+                    ts = time.monotonic()
+                    s4_facts, s4_edges = self._stage4.extract(text, source_window_id)
+                    stage_latency[4] = (time.monotonic() - ts) * 1000
+                    all_facts.extend(s4_facts)
+                    all_edges.extend(s4_edges)
+                    result.stages_run.append(4)
+                    result.stage_yields[4] = len(s4_facts)
+                    result.escalation_triggers.append(
+                        f"stage_3_relation_yield={relation_per_sent:.2f} < 0.1"
+                    )
+                else:
+                    result.stages_skipped.append(4)
+            else:
+                result.stages_skipped.append(4)
+        else:
+            result.stages_skipped.append(4)
+
+        # ── STAGE 5: Discourse (conditional on content type) ──────────────
+        if not _short_circuited and self._enable_5 and content_type in (ContentType.REASONING_DENSE, ContentType.NARRATIVE):
+            ts = time.monotonic()
+            s5_facts, s5_edges = self._stage5.extract(text, source_window_id)
+            stage_latency[5] = (time.monotonic() - ts) * 1000
+            all_facts.extend(s5_facts)
+            all_edges.extend(s5_edges)
+            result.stages_run.append(5)
+            result.stage_yields[5] = len(s5_facts)
+        else:
+            result.stages_skipped.append(5)
+
+        # ── STAGE 6: LLM (conditional — rare) ────────────────────────────
+        if (
+            not _short_circuited
+            and self._enable_6
+            and content_type == ContentType.REASONING_DENSE
+            and self._stage6.is_available
+        ):
+            # Check Stage 5 edge yield
+            _sent_count = max(len(text.split(".")), 1)
+            s5_edge_yield = len([e for e in all_edges if e.source_stage == 5]) / _sent_count
+            if s5_edge_yield < 0.1:
+                ts = time.monotonic()
+                s6_facts, s6_edges = self._stage6.extract(text, source_window_id)
+                stage_latency[6] = (time.monotonic() - ts) * 1000
+                all_facts.extend(s6_facts)
+                all_edges.extend(s6_edges)
+                result.stages_run.append(6)
+                result.stage_yields[6] = len(s6_facts)
+                result.escalation_triggers.append(
+                    f"stage_5_edge_yield={s5_edge_yield:.2f} < 0.1"
+                )
+            else:
+                result.stages_skipped.append(6)
+        else:
+            result.stages_skipped.append(6)
+
+        # -- Tick Stage 3 idle counter --------------------------------------
+        if 3 not in result.stages_run:
+            self._stage3.tick_idle()
+
+        # -- Assemble result ------------------------------------------------
+        result.facts = all_facts
+        result.edges = all_edges
+        result.per_stage_latency = stage_latency
+        result.entity_density = len(s1_facts) / max(word_count, 1)
+        result.total_extraction_latency_ms = (time.monotonic() - t0) * 1000
+        result.finalize()
+
+        # -- Record for calibration -----------------------------------------
+        self._calibration.record(result)
+
+        return result

crp/extraction/quality_gate.py

@@ -0,0 +1,237 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Post-extraction quality gate — 3-tier validation (§3.2 2H).
+
+Tier 1: Structural validation (schema, parse success, empty facts).
+Tier 2: Confidence threshold filter (flag low-confidence facts).
+Tier 3: Anomaly detection (fact explosion, zero facts, duplicates).
+"""
+
+from __future__ import annotations
+
+import logging
+import statistics
+from typing import Any
+
+from crp.extraction.types import (
+    ExtractionResult,
+    Fact,
+    ValidationIssue,
+    ValidationResult,
+    ValidationSeverity,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Tier 1 — Structural Validation
+# ---------------------------------------------------------------------------
+
+def structural_validation(
+    result: ExtractionResult,
+    output_schema: dict[str, Any] | None = None,
+) -> ValidationResult:
+    """Check schema compliance, parse success rate, and empty facts."""
+    issues: list[ValidationIssue] = []
+
+    # Check 1: JSON Schema compliance (if output_schema provided)
+    if output_schema:
+        try:
+            import jsonschema  # type: ignore[import-untyped]
+
+            payload = [{"text": f.text, "category": f.category} for f in result.facts]
+            jsonschema.validate(payload, output_schema)
+        except ImportError:
+            pass  # jsonschema optional
+        except Exception as exc:
+            issues.append(ValidationIssue(
+                type="SCHEMA_MISMATCH",
+                severity=ValidationSeverity.HIGH,
+                detail=str(exc)[:200],
+            ))
+
+    # Check 2: Parsing success rate — count facts with very short text
+    parse_failures = sum(1 for f in result.facts if not f.text.strip())
+    if result.total_facts > 0 and parse_failures > 0.1 * result.total_facts:
+        issues.append(ValidationIssue(
+            type="HIGH_PARSE_FAILURE_RATE",
+            severity=ValidationSeverity.MEDIUM,
+            detail=f"{parse_failures} facts could not be parsed",
+        ))
+
+    # Check 3: Suspiciously short facts
+    empty = [f for f in result.facts if len(f.text.strip()) < 5]
+    if empty:
+        issues.append(ValidationIssue(
+            type="EMPTY_FACTS",
+            severity=ValidationSeverity.LOW,
+            detail=f"{len(empty)} facts are suspiciously short (<5 chars)",
+        ))
+
+    passed = all(i.severity != ValidationSeverity.HIGH for i in issues)
+    return ValidationResult(tier=1, passed=passed, issues=issues)
+
+
+# ---------------------------------------------------------------------------
+# Tier 2 — Confidence Threshold
+# ---------------------------------------------------------------------------
+
+def confidence_threshold_filter(
+    result: ExtractionResult,
+    floor: float = 0.6,
+) -> ValidationResult:
+    """Mark facts below the confidence floor as flagged.
+
+    Low-confidence facts are NOT excluded — they remain but get a
+    0.5× score multiplier in envelope packing.
+    """
+    flagged = 0
+    for fact in result.facts:
+        if fact.confidence < floor:
+            fact.flagged_confidence = True
+            fact.confidence_flag_reason = f"Below baseline {floor:.2f}"
+            flagged += 1
+
+    issues: list[ValidationIssue] = []
+    if flagged:
+        issues.append(ValidationIssue(
+            type="LOW_CONFIDENCE_FACTS",
+            severity=ValidationSeverity.LOW,
+            detail=f"{flagged} facts below confidence floor {floor:.2f}",
+        ))
+    return ValidationResult(tier=2, passed=True, issues=issues)
+
+
+# ---------------------------------------------------------------------------
+# Tier 3 — Anomaly Detection
+# ---------------------------------------------------------------------------
+
+def anomaly_detection(
+    result: ExtractionResult,
+    history: list[ExtractionResult] | None = None,
+) -> ValidationResult:
+    """Check for fact-count explosion, zero facts, and duplicates."""
+    issues: list[ValidationIssue] = []
+
+    # Check 1: Fact count explosion (>5× typical)
+    if history and len(history) >= 2:
+        typical = statistics.mean([r.total_facts for r in history[-5:]])
+        if typical > 0 and result.total_facts > 5 * typical:
+            issues.append(ValidationIssue(
+                type="UNUSUALLY_HIGH_FACT_COUNT",
+                severity=ValidationSeverity.MEDIUM,
+                detail=(
+                    f"{result.total_facts} facts vs typical ~{typical:.0f}. "
+                    "Possible extraction running on stack trace?"
+                ),
+            ))
+
+    # Check 2: Zero facts
+    if result.total_facts == 0:
+        issues.append(ValidationIssue(
+            type="NO_FACTS_EXTRACTED",
+            severity=ValidationSeverity.MEDIUM,
+            detail="Zero facts extracted — content may be too short or unstructured",
+        ))
+
+    # Check 3: >20% duplicates
+    hashes = [hash(f.text) for f in result.facts]
+    duplicates = len(hashes) - len(set(hashes))
+    if result.total_facts > 0 and duplicates > 0.2 * result.total_facts:
+        issues.append(ValidationIssue(
+            type="EXCESSIVE_NEAR_DUPLICATES",
+            severity=ValidationSeverity.LOW,
+            detail=f"{duplicates} near-duplicate facts detected",
+        ))
+
+    passed = all(i.severity != ValidationSeverity.HIGH for i in issues)
+    return ValidationResult(tier=3, passed=passed, issues=issues)
+
+
+# ---------------------------------------------------------------------------
+# Fact normalisation
+# ---------------------------------------------------------------------------
+
+def normalize_facts(facts: list[Fact], max_tokens: int = 100, min_tokens: int = 5) -> list[Fact]:
+    """Split overly long facts and merge very short ones.
+
+    Uses word count as a proxy for token count (accurate tokenisation is Phase 4).
+    """
+    normalized: list[Fact] = []
+    short_buffer: list[Fact] = []
+
+    for fact in facts:
+        words = fact.text.split()
+        wc = len(words)
+
+        if wc > max_tokens:
+            # Split into chunks of ~max_tokens words
+            for i in range(0, wc, max_tokens):
+                chunk = " ".join(words[i : i + max_tokens])
+                normalized.append(Fact(
+                    text=chunk,
+                    category=fact.category,
+                    source_window_id=fact.source_window_id,
+                    confidence=fact.confidence,
+                    extraction_stage=fact.extraction_stage,
+                    metadata={**fact.metadata, "split_from": fact.id},
+                ))
+        elif wc < min_tokens:
+            short_buffer.append(fact)
+            if len(short_buffer) >= 3:
+                merged_text = " | ".join(f.text for f in short_buffer)
+                normalized.append(Fact(
+                    text=merged_text,
+                    category=short_buffer[0].category,
+                    source_window_id=short_buffer[0].source_window_id,
+                    confidence=min(f.confidence for f in short_buffer),
+                    extraction_stage=short_buffer[0].extraction_stage,
+                    metadata={"merged_from": [f.id for f in short_buffer]},
+                ))
+                short_buffer.clear()
+        else:
+            normalized.append(fact)
+
+    # Flush remaining short facts
+    normalized.extend(short_buffer)
+    return normalized
+
+
+# ---------------------------------------------------------------------------
+# Composite quality gate
+# ---------------------------------------------------------------------------
+
+def run_quality_gate(
+    result: ExtractionResult,
+    *,
+    output_schema: dict[str, Any] | None = None,
+    confidence_floor: float = 0.6,
+    history: list[ExtractionResult] | None = None,
+) -> ExtractionResult:
+    """Run all 3 quality-gate tiers and update the ExtractionResult in-place."""
+    all_issues: list[str] = []
+
+    # Tier 1
+    v1 = structural_validation(result, output_schema)
+    if not v1.passed:
+        result.quality_gate_passed = False
+    all_issues.extend(f"[T1] {i.type}: {i.detail}" for i in v1.issues)
+
+    # Tier 2
+    v2 = confidence_threshold_filter(result, confidence_floor)
+    all_issues.extend(f"[T2] {i.type}: {i.detail}" for i in v2.issues)
+
+    # Tier 3
+    v3 = anomaly_detection(result, history)
+    if not v3.passed:
+        result.quality_gate_passed = False
+    all_issues.extend(f"[T3] {i.type}: {i.detail}" for i in v3.issues)
+
+    result.quality_issues = all_issues
+
+    # Normalise facts
+    result.facts = normalize_facts(result.facts)
+    result.facts_after_normalization = len(result.facts)
+
+    return result