crprotocol 2.0.0__py3-none-any.whl

crp/extraction/stage4_uie.py

@@ -0,0 +1,183 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Stage 4 — UIE relational extraction (SHOULD, ~100ms, lazy model load).
+
+Extracts (subject, predicate, object) triples and converts them to FactEdge
+records. Trigger: Stage 3 relation yield < 0.1 per sentence.
+Model: UIE / universal IE (~400MB), loaded lazily.
+Graceful fallback: returns empty if unavailable.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Protocol, runtime_checkable
+
+from crp.extraction.types import Fact, FactEdge, RelationType
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# UIE model protocol
+# ---------------------------------------------------------------------------
+
+@runtime_checkable
+class UIEModel(Protocol):
+    """Minimal interface for a Universal Information Extraction model."""
+
+    def extract_triples(
+        self, text: str
+    ) -> list[dict[str, Any]]:
+        """Return list of dicts with keys: subject, predicate, object, confidence."""
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Triple → FactEdge mapping
+# ---------------------------------------------------------------------------
+
+_PREDICATE_TO_RELATION: dict[str, RelationType] = {
+    "causes": RelationType.CAUSE_EFFECT,
+    "caused by": RelationType.CAUSE_EFFECT,
+    "leads to": RelationType.CONSEQUENCE,
+    "results in": RelationType.CONSEQUENCE,
+    "depends on": RelationType.CONDITION_FOR,
+    "requires": RelationType.CONDITION_FOR,
+    "contrasts with": RelationType.CONTRAST,
+    "despite": RelationType.CONCESSION,
+    "elaborates": RelationType.ELABORATION,
+    "extends": RelationType.ELABORATION,
+    "follows": RelationType.SEQUENCE,
+    "precedes": RelationType.SEQUENCE,
+}
+
+
+def _map_predicate(predicate: str) -> RelationType:
+    """Best-effort mapping from free-text predicate to RelationType."""
+    pred_lower = predicate.strip().lower()
+    for key, rel in _PREDICATE_TO_RELATION.items():
+        if key in pred_lower:
+            return rel
+    return RelationType.RELATED
+
+
+# ---------------------------------------------------------------------------
+# Stage 4 Extractor
+# ---------------------------------------------------------------------------
+
+class UIEExtractor:
+    """Stage 4 — UIE triple extraction (lazy, optional).
+
+    Loads the model on first use. Returns ``(facts, edges)`` where *facts*
+    are the subject/object entities and *edges* are the relations.
+    If the UIE library is unavailable, all calls return ``([], [])``.
+    """
+
+    def __init__(self) -> None:
+        self._model: UIEModel | None = None
+        self._available: bool | None = None
+
+    # -- Lifecycle ----------------------------------------------------------
+
+    def _ensure_model(self) -> UIEModel | None:
+        if self._available is False:
+            return None
+        if self._model is not None:
+            return self._model
+        try:
+            # Try to import a UIE implementation.
+            # The spec is model-agnostic; accept any class exposing extract_triples().
+            from uie import UIE  # type: ignore[import-untyped]
+
+            self._model = UIE()  # type: ignore[assignment]
+            self._available = True
+            logger.info("UIE model loaded successfully")
+            return self._model
+        except Exception:
+            self._available = False
+            logger.warning("UIE not available — Stage 4 will be skipped")
+            return None
+
+    def unload(self) -> None:
+        self._model = None
+
+    @property
+    def is_available(self) -> bool:
+        if self._available is None:
+            self._ensure_model()
+        return self._available is True
+
+    # -- Extraction ---------------------------------------------------------
+
+    def extract(
+        self,
+        text: str,
+        source_window_id: str = "",
+    ) -> tuple[list[Fact], list[FactEdge]]:
+        """Extract relational triples from *text*.
+
+        Returns ``(facts, edges)`` — each triple yields two Fact items
+        (subject, object) and one FactEdge.
+        Returns ``([], [])`` on failure or if model unavailable.
+        """
+        model = self._ensure_model()
+        if model is None:
+            return [], []
+
+        try:
+            triples = model.extract_triples(text)
+        except Exception:
+            logger.exception("UIE extraction failed")
+            return [], []
+
+        facts: list[Fact] = []
+        edges: list[FactEdge] = []
+        seen_texts: dict[str, str] = {}  # text → fact_id (dedup entities)
+
+        for triple in triples:
+            subj_text = str(triple.get("subject", ""))
+            obj_text = str(triple.get("object", ""))
+            predicate = str(triple.get("predicate", ""))
+            conf = float(triple.get("confidence", 0.70))
+
+            if not subj_text or not obj_text:
+                continue
+
+            # Dedup entity facts
+            if subj_text not in seen_texts:
+                subj_fact = Fact(
+                    text=subj_text,
+                    category="uie_entity",
+                    source_window_id=source_window_id,
+                    confidence=min(0.85, max(0.70, conf)),
+                    extraction_stage=4,
+                    metadata={"role": "subject"},
+                )
+                facts.append(subj_fact)
+                seen_texts[subj_text] = subj_fact.id
+            subj_id = seen_texts[subj_text]
+
+            if obj_text not in seen_texts:
+                obj_fact = Fact(
+                    text=obj_text,
+                    category="uie_entity",
+                    source_window_id=source_window_id,
+                    confidence=min(0.85, max(0.70, conf)),
+                    extraction_stage=4,
+                    metadata={"role": "object"},
+                )
+                facts.append(obj_fact)
+                seen_texts[obj_text] = obj_fact.id
+            obj_id = seen_texts[obj_text]
+
+            edges.append(FactEdge(
+                source_id=subj_id,
+                target_id=obj_id,
+                relation_type=_map_predicate(predicate),
+                confidence=min(0.85, max(0.70, conf)),
+                source_stage=4,
+                metadata={"predicate": predicate},
+            ))
+
+        return facts, edges

crp/extraction/stage5_discourse.py

@@ -0,0 +1,175 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Stage 5 — Discourse structure extraction (SHOULD, ~150ms, CPU-only).
+
+Detects discourse markers and maps them to semantic relation types (RST-inspired).
+Trigger: content_type in {REASONING_DENSE, NARRATIVE}.
+No ML model — pure pattern matching over sentences.
+"""
+
+from __future__ import annotations
+
+import re
+
+from crp.extraction.types import Fact, FactEdge, RelationType
+
+# ---------------------------------------------------------------------------
+# Discourse marker → relation-type mapping
+# ---------------------------------------------------------------------------
+
+_MARKER_GROUPS: list[tuple[RelationType, list[str]]] = [
+    (RelationType.CONDITION_FOR, [
+        "if", "unless", "provided", "provided that", "assuming",
+        "given that", "in case", "on condition that",
+    ]),
+    (RelationType.CAUSE_EFFECT, [
+        "because", "since", "due to", "owing to", "causes",
+        "caused by", "as a result of",
+    ]),
+    (RelationType.CONTRAST, [
+        "however", "but", "yet", "on the other hand",
+        "in contrast", "conversely", "nevertheless",
+    ]),
+    (RelationType.CONCESSION, [
+        "although", "despite", "even though", "in spite of",
+        "notwithstanding", "regardless",
+    ]),
+    (RelationType.CONSEQUENCE, [
+        "therefore", "thus", "hence", "so", "consequently",
+        "as a result", "accordingly",
+    ]),
+    (RelationType.ELABORATION, [
+        "as", "given", "for example", "for instance",
+        "in particular", "specifically", "namely",
+        "that is", "i.e.", "e.g.",
+    ]),
+    (RelationType.SEQUENCE, [
+        "and then", "subsequently", "next", "afterwards",
+        "following", "before", "after", "finally",
+        "first", "second", "third", "lastly",
+    ]),
+]
+
+# Build compiled regex per group — match markers at word boundaries
+_COMPILED_MARKERS: list[tuple[RelationType, re.Pattern[str]]] = []
+for _rel, _markers in _MARKER_GROUPS:
+    # Sort longest first so "provided that" matches before "provided"
+    _sorted = sorted(_markers, key=len, reverse=True)
+    escaped = [re.escape(m) for m in _sorted]
+    _COMPILED_MARKERS.append((
+        _rel,
+        re.compile(r"\b(?:" + "|".join(escaped) + r")\b", re.IGNORECASE),
+    ))
+
+# Flat set for quick counting
+_ALL_MARKERS_FLAT: set[str] = set()
+for _, markers in _MARKER_GROUPS:
+    _ALL_MARKERS_FLAT.update(markers)
+
+# ---------------------------------------------------------------------------
+# Sentence splitting (reuse from stage2 would be nice, but keep self-contained)
+# ---------------------------------------------------------------------------
+
+_SENT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z])")
+
+
+def _split_sentences(text: str) -> list[str]:
+    raw = _SENT_RE.split(text.strip())
+    return [s.strip() for s in raw if len(s.strip()) > 5]
+
+
+# ---------------------------------------------------------------------------
+# Public helpers (used by complexity detector)
+# ---------------------------------------------------------------------------
+
+def count_discourse_markers(text: str) -> int:
+    """Count total discourse-marker occurrences in *text* (fast)."""
+    total = 0
+    text_lower = text.lower()
+    for marker in _ALL_MARKERS_FLAT:
+        # Word-boundary check via regex on each marker would be expensive;
+        # for the counting use-case a simple substring is sufficient.
+        start = 0
+        while True:
+            idx = text_lower.find(marker, start)
+            if idx == -1:
+                break
+            # Rough word-boundary check
+            before_ok = idx == 0 or not text_lower[idx - 1].isalnum()
+            after_idx = idx + len(marker)
+            after_ok = after_idx >= len(text_lower) or not text_lower[after_idx].isalnum()
+            if before_ok and after_ok:
+                total += 1
+            start = idx + 1
+    return total
+
+
+# ---------------------------------------------------------------------------
+# Stage 5 Extractor
+# ---------------------------------------------------------------------------
+
+class DiscourseExtractor:
+    """Stage 5 — discourse-structure extraction (CPU-only)."""
+
+    def extract(
+        self,
+        text: str,
+        source_window_id: str = "",
+    ) -> tuple[list[Fact], list[FactEdge]]:
+        """Detect discourse markers and create FactEdge relations.
+
+        Returns ``(marker_facts, edges)`` where *marker_facts* are the
+        clauses surrounding each detected marker, and *edges* link them.
+        """
+        sentences = _split_sentences(text)
+        if not sentences:
+            return [], []
+
+        facts: list[Fact] = []
+        edges: list[FactEdge] = []
+        sent_fact_ids: dict[int, str] = {}  # sentence_index → fact_id
+
+        def _get_or_create_fact(idx: int) -> str:
+            """Ensure a Fact exists for the sentence at *idx*."""
+            if idx in sent_fact_ids:
+                return sent_fact_ids[idx]
+            f = Fact(
+                text=sentences[idx],
+                category="discourse_unit",
+                source_window_id=source_window_id,
+                confidence=0.70,
+                extraction_stage=5,
+                metadata={"sentence_index": idx},
+            )
+            facts.append(f)
+            sent_fact_ids[idx] = f.id
+            return f.id
+
+        for i, sent in enumerate(sentences):
+            for rel_type, pattern in _COMPILED_MARKERS:
+                match = pattern.search(sent)
+                if match is None:
+                    continue
+
+                # The marker links this sentence (or the clause after the marker)
+                # to the preceding sentence.
+                before_idx = max(0, i - 1)
+                after_idx = i
+
+                # Avoid self-loops
+                if before_idx == after_idx and i == 0:
+                    continue
+
+                before_id = _get_or_create_fact(before_idx)
+                after_id = _get_or_create_fact(after_idx)
+
+                edges.append(FactEdge(
+                    source_id=before_id,
+                    target_id=after_id,
+                    relation_type=rel_type,
+                    confidence=0.70,
+                    source_stage=5,
+                    metadata={"marker": match.group(0), "sentence_index": i},
+                ))
+
+        return facts, edges

crp/extraction/stage6_llm.py

@@ -0,0 +1,178 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Stage 6 — LLM-assisted relational extraction (MAY, expensive).
+
+Dispatches a dedicated extraction window to a small LLM to extract logical
+relationships from reasoning-dense content. Trigger: content_type == REASONING_DENSE
+AND Stage 5 edge_yield < 0.1 edges/sentence.
+
+This stage is **user-configurable** (can be disabled via config flag).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from collections.abc import Callable
+
+from crp.extraction.types import Fact, FactEdge, RelationType
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Extraction prompt
+# ---------------------------------------------------------------------------
+
+_EXTRACTION_PROMPT = (
+    "Extract ALL logical relationships, conditions, dependencies, "
+    "and reasoning chains from this text. Output as a JSON array of objects "
+    'with keys: "subject", "predicate", "object". Example:\n'
+    '[{"subject": "A", "predicate": "causes", "object": "B"}]'
+)
+
+_RELATION_MAP: dict[str, RelationType] = {
+    "causes": RelationType.CAUSE_EFFECT,
+    "caused by": RelationType.CAUSE_EFFECT,
+    "leads to": RelationType.CONSEQUENCE,
+    "results in": RelationType.CONSEQUENCE,
+    "depends on": RelationType.CONDITION_FOR,
+    "requires": RelationType.CONDITION_FOR,
+    "if": RelationType.CONDITION_FOR,
+    "contrasts": RelationType.CONTRAST,
+    "despite": RelationType.CONCESSION,
+    "elaborates": RelationType.ELABORATION,
+    "follows": RelationType.SEQUENCE,
+}
+
+# ---------------------------------------------------------------------------
+# Dispatch callback type
+# ---------------------------------------------------------------------------
+
+# Type alias for the LLM dispatch function the pipeline will inject.
+# Signature: dispatch(system_prompt, task_input, max_output_tokens) → str
+DispatchFn = Callable[[str, str, int], str]
+
+
+# ---------------------------------------------------------------------------
+# Response parser
+# ---------------------------------------------------------------------------
+
+_JSON_ARRAY_RE = re.compile(r"\[.*\]", re.DOTALL)
+
+
+def _parse_extraction_response(raw: str) -> list[dict[str, str]]:
+    """Best-effort JSON parse from LLM output."""
+    m = _JSON_ARRAY_RE.search(raw)
+    if not m:
+        logger.debug("Stage 6: no JSON array found in LLM response (%d chars)", len(raw))
+        return []
+    try:
+        data = json.loads(m.group(0))
+        if isinstance(data, list):
+            return [d for d in data if isinstance(d, dict)]
+    except (json.JSONDecodeError, ValueError):
+        logger.debug("Stage 6: JSON parse failed for extraction response")
+    return []
+
+
+def _map_predicate(predicate: str) -> RelationType:
+    pred = predicate.strip().lower()
+    for key, rel in _RELATION_MAP.items():
+        if key in pred:
+            return rel
+    return RelationType.RELATED
+
+
+# ---------------------------------------------------------------------------
+# Stage 6 Extractor
+# ---------------------------------------------------------------------------
+
+class LLMExtractor:
+    """Stage 6 — LLM-assisted extraction (optional, expensive).
+
+    Requires a *dispatch_fn* to be injected by the pipeline. If not set,
+    ``extract()`` returns empty.
+    """
+
+    def __init__(self, dispatch_fn: DispatchFn | None = None) -> None:
+        self._dispatch = dispatch_fn
+
+    @property
+    def is_available(self) -> bool:
+        return self._dispatch is not None
+
+    def set_dispatch(self, fn: DispatchFn) -> None:
+        self._dispatch = fn
+
+    def extract(
+        self,
+        text: str,
+        source_window_id: str = "",
+        max_input_chars: int = 8000,
+        max_output_tokens: int = 1024,
+    ) -> tuple[list[Fact], list[FactEdge]]:
+        """Dispatch extraction window and parse results.
+
+        Returns ``(facts, edges)`` or ``([], [])`` if dispatch unavailable.
+        """
+        if self._dispatch is None:
+            return [], []
+
+        # Chunk if necessary
+        chunk = text[:max_input_chars]
+
+        try:
+            raw = self._dispatch(_EXTRACTION_PROMPT, chunk, max_output_tokens)
+        except Exception:
+            logger.exception("Stage 6 LLM dispatch failed")
+            return [], []
+
+        triples = _parse_extraction_response(raw)
+        if not triples:
+            return [], []
+
+        facts: list[Fact] = []
+        edges: list[FactEdge] = []
+        seen: dict[str, str] = {}
+
+        for triple in triples:
+            subj = str(triple.get("subject", "")).strip()
+            obj = str(triple.get("object", "")).strip()
+            pred = str(triple.get("predicate", "")).strip()
+            if not subj or not obj:
+                continue
+
+            if subj not in seen:
+                sf = Fact(
+                    text=subj,
+                    category="llm_entity",
+                    source_window_id=source_window_id,
+                    confidence=0.75,
+                    extraction_stage=6,
+                    metadata={"role": "subject"},
+                )
+                facts.append(sf)
+                seen[subj] = sf.id
+            if obj not in seen:
+                of = Fact(
+                    text=obj,
+                    category="llm_entity",
+                    source_window_id=source_window_id,
+                    confidence=0.75,
+                    extraction_stage=6,
+                    metadata={"role": "object"},
+                )
+                facts.append(of)
+                seen[obj] = of.id
+
+            edges.append(FactEdge(
+                source_id=seen[subj],
+                target_id=seen[obj],
+                relation_type=_map_predicate(pred),
+                confidence=0.75,
+                source_stage=6,
+                metadata={"predicate": pred},
+            ))
+
+        return facts, edges