crprotocol 2.0.0__py3-none-any.whl

crp/extraction/stage1_regex.py

@@ -0,0 +1,173 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Stage 1 — Regex extraction (~1ms, MUST).
+
+Extracts structured entities: IPs, CVEs, URLs, emails, JSON blocks,
+error codes, versions, ports, and cryptographic hashes.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from crp.extraction.types import Fact
+
+# ---------------------------------------------------------------------------
+# Pattern registry
+# ---------------------------------------------------------------------------
+
+@dataclass(frozen=True)
+class RegexPattern:
+    """A single extraction pattern."""
+
+    name: str
+    pattern: re.Pattern[str]
+    category: str
+    confidence: float = 0.95
+
+
+# Compiled patterns — order matters for priority in case of overlapping spans.
+_BUILTIN_PATTERNS: list[RegexPattern] = [
+    RegexPattern(
+        name="ipv4_address",
+        pattern=re.compile(
+            r"\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}"
+            r"(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b"
+        ),
+        category="network_entity",
+    ),
+    RegexPattern(
+        name="ipv6_address",
+        pattern=re.compile(
+            r"\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b"
+            r"|"
+            r"\b(?:[0-9a-fA-F]{1,4}:){1,7}:\b"
+            r"|"
+            r"\b::(?:[0-9a-fA-F]{1,4}:){0,5}[0-9a-fA-F]{1,4}\b"
+        ),
+        category="network_entity",
+    ),
+    RegexPattern(
+        name="cve_id",
+        pattern=re.compile(r"\bCVE-\d{4}-\d{4,7}\b"),
+        category="vulnerability",
+    ),
+    RegexPattern(
+        name="url",
+        pattern=re.compile(
+            r"https?://[^\s\"'<>\])}]+",
+            re.ASCII,
+        ),
+        category="resource",
+    ),
+    RegexPattern(
+        name="email",
+        pattern=re.compile(
+            r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b"
+        ),
+        category="contact",
+    ),
+    RegexPattern(
+        name="json_block",
+        pattern=re.compile(r"\{[^{}]{10,}\}", re.DOTALL),
+        category="structured_data",
+    ),
+    RegexPattern(
+        name="error_code",
+        pattern=re.compile(
+            r"\b(?:ERR|ERROR|WARN|FATAL|CRITICAL|E|W)[-_]?\d{3,6}\b",
+            re.IGNORECASE,
+        ),
+        category="error_code",
+    ),
+    RegexPattern(
+        name="semver",
+        pattern=re.compile(
+            r"\bv?\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.]+)?\b"
+        ),
+        category="version",
+    ),
+    RegexPattern(
+        name="port",
+        pattern=re.compile(
+            r"\b(?:port\s+|:)(\d{1,5})\b", re.IGNORECASE
+        ),
+        category="network_endpoint",
+    ),
+    RegexPattern(
+        name="hash_sha256",
+        pattern=re.compile(r"\b[0-9a-fA-F]{64}\b"),
+        category="identifier",
+    ),
+    RegexPattern(
+        name="hash_md5",
+        pattern=re.compile(r"\b[0-9a-fA-F]{32}\b"),
+        category="identifier",
+    ),
+]
+
+
+class RegexExtractor:
+    """Stage 1 — fast regex extraction with extensible pattern registry."""
+
+    def __init__(self) -> None:
+        self._patterns: list[RegexPattern] = list(_BUILTIN_PATTERNS)
+        self._custom_patterns: list[RegexPattern] = []
+
+    # -- Public API ---------------------------------------------------------
+
+    def register_pattern(
+        self,
+        name: str,
+        pattern: str | re.Pattern[str],
+        category: str,
+        confidence: float = 0.90,
+    ) -> None:
+        """Add a user-defined extraction pattern."""
+        compiled = re.compile(pattern) if isinstance(pattern, str) else pattern
+        rp = RegexPattern(name=name, pattern=compiled, category=category, confidence=confidence)
+        self._custom_patterns.append(rp)
+
+    def extract(
+        self,
+        text: str,
+        source_window_id: str = "",
+    ) -> list[Fact]:
+        """Extract all regex-matched facts from *text*.
+
+        Returns de-duplicated facts ordered by position in text.
+        """
+        seen_spans: set[tuple[int, int]] = set()
+        facts: list[tuple[int, Fact]] = []  # (start_pos, Fact)
+
+        for rp in self._all_patterns:
+            for m in rp.pattern.finditer(text):
+                span = (m.start(), m.end())
+                if span in seen_spans:
+                    continue
+                seen_spans.add(span)
+                matched = m.group(0)
+                facts.append((
+                    m.start(),
+                    Fact(
+                        text=matched,
+                        category=rp.category,
+                        source_window_id=source_window_id,
+                        confidence=rp.confidence,
+                        extraction_stage=1,
+                        metadata={"pattern": rp.name, "span": list(span)},
+                    ),
+                ))
+
+        # Stable sort by position
+        facts.sort(key=lambda t: t[0])
+        return [f for _, f in facts]
+
+    @property
+    def _all_patterns(self) -> list[RegexPattern]:
+        return self._patterns + self._custom_patterns
+
+    @property
+    def pattern_count(self) -> int:
+        return len(self._all_patterns)

crp/extraction/stage2_statistical.py

@@ -0,0 +1,244 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Stage 2 — Statistical NLP extraction (~5ms, MUST).
+
+Pure-Python: TextRank key-sentence extraction, noun-phrase heuristics,
+section-header detection, list-item extraction, and numerical-value extraction.
+No ML model dependencies.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+
+from crp.extraction.types import Fact
+
+# ---------------------------------------------------------------------------
+# Sentence splitting
+# ---------------------------------------------------------------------------
+
+_SENT_SPLIT = re.compile(r"(?<=[.!?])\s+(?=[A-Z])")
+_WORD_SPLIT = re.compile(r"\w+", re.UNICODE)
+
+# Simple stop-word set (top ~100 English)
+_STOP_WORDS: frozenset[str] = frozenset(
+    ["a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "aren't", "as", "at", "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "can't", "cannot", "could", "couldn't", "did", "didn't", "do", "does", "doesn't", "doing", "don't", "down", "during", "each", "few", "for", "from", "further", "get", "got", "had", "hadn't", "has", "hasn't", "have", "haven't", "having", "he", "her", "here", "hers", "herself", "him", "himself", "his", "how", "i", "if", "in", "into", "is", "isn't", "it", "its", "itself", "just", "me", "more", "most", "mustn't", "my", "myself", "no", "nor", "not", "of", "off", "on", "once", "only", "or", "other", "our", "ours", "ourselves", "out", "over", "own", "same", "she", "should", "shouldn't", "so", "some", "such", "than", "that", "the", "their", "theirs", "them", "themselves", "then", "there", "these", "they", "this", "those", "through", "to", "too", "under", "until", "up", "very", "was", "wasn't", "we", "were", "weren't", "what", "when", "where", "which", "while", "who", "whom", "why", "will", "with", "won't", "would", "wouldn't", "you", "your", "yours", "yourself", "yourselves"]
+)
+
+
+def _sentences(text: str) -> list[str]:
+    """Split text into sentences."""
+    raw = _SENT_SPLIT.split(text.strip())
+    return [s.strip() for s in raw if len(s.strip()) > 5]
+
+
+def _words(text: str) -> list[str]:
+    return _WORD_SPLIT.findall(text.lower())
+
+
+def _content_words(text: str) -> list[str]:
+    return [w for w in _words(text) if w not in _STOP_WORDS and len(w) > 2]
+
+
+# ---------------------------------------------------------------------------
+# TextRank (simplified, no external deps)
+# ---------------------------------------------------------------------------
+
+def _sentence_similarity(s1_words: list[str], s2_words: list[str]) -> float:
+    """Jaccard-like overlap between content-word sets."""
+    if not s1_words or not s2_words:
+        return 0.0
+    set1, set2 = set(s1_words), set(s2_words)
+    overlap = len(set1 & set2)
+    return overlap / (math.log(len(set1) + 1) + math.log(len(set2) + 1) + 1e-9)
+
+
+def textrank_sentences(
+    sentences: list[str],
+    top_k: int = 5,
+    damping: float = 0.85,
+    iterations: int = 30,
+    convergence: float = 1e-4,
+) -> list[tuple[int, float]]:
+    """Return indices + scores of top-K sentences via TextRank.
+
+    Steps:
+    1. Build sentence similarity graph.
+    2. Run PageRank-style iteration.
+    3. Return top-K by score (original order preserved).
+    """
+    n = len(sentences)
+    if n == 0:
+        return []
+    if n <= top_k:
+        return [(i, 1.0) for i in range(n)]
+
+    # Tokenise once
+    tokenised = [_content_words(s) for s in sentences]
+
+    # Build adjacency weights
+    weights: list[list[float]] = [[0.0] * n for _ in range(n)]
+    for i in range(n):
+        for j in range(i + 1, n):
+            sim = _sentence_similarity(tokenised[i], tokenised[j])
+            weights[i][j] = sim
+            weights[j][i] = sim
+
+    # Row sums for normalisation
+    row_sums = [sum(row) for row in weights]
+
+    # PageRank iteration
+    scores = [1.0 / n] * n
+    for _ in range(iterations):
+        new_scores = [0.0] * n
+        max_delta = 0.0
+        for i in range(n):
+            rank_sum = 0.0
+            for j in range(n):
+                if row_sums[j] > 0:
+                    rank_sum += weights[j][i] / row_sums[j] * scores[j]
+            new_scores[i] = (1 - damping) / n + damping * rank_sum
+            max_delta = max(max_delta, abs(new_scores[i] - scores[i]))
+        scores = new_scores
+        if max_delta < convergence:
+            break
+
+    # Top-K indices sorted by score descending, then stable-sort by original position
+    ranked = sorted(range(n), key=lambda i: scores[i], reverse=True)[:top_k]
+    ranked.sort()  # Restore document order
+    return [(i, scores[i]) for i in ranked]
+
+
+# ---------------------------------------------------------------------------
+# Noun-phrase heuristic (regex, no POS tagger)
+# ---------------------------------------------------------------------------
+
+_NP_PATTERN = re.compile(
+    r"\b(?:[A-Z][a-z]+(?:\s+(?:of|the|and|for|in|on|with|to)\s+)?){1,4}"
+    r"[A-Z][a-z]+\b"
+)
+
+
+def _extract_noun_phrases(text: str) -> list[str]:
+    """Extract capitalised multi-word noun phrases."""
+    return list({m.group(0) for m in _NP_PATTERN.finditer(text)})
+
+
+# ---------------------------------------------------------------------------
+# Section headers
+# ---------------------------------------------------------------------------
+
+_MARKDOWN_HEADER = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE)
+_UNDERLINE_HEADER = re.compile(r"^(.+)\n[=\-]{3,}$", re.MULTILINE)
+
+
+def _extract_headers(text: str) -> list[str]:
+    headers: list[str] = []
+    for m in _MARKDOWN_HEADER.finditer(text):
+        headers.append(m.group(2).strip())
+    for m in _UNDERLINE_HEADER.finditer(text):
+        headers.append(m.group(1).strip())
+    return headers
+
+
+# ---------------------------------------------------------------------------
+# List items
+# ---------------------------------------------------------------------------
+
+_LIST_ITEM = re.compile(
+    r"^[ \t]*(?:[-*+]|\d+[.)])\s+(.+)$", re.MULTILINE
+)
+
+
+def _extract_list_items(text: str) -> list[str]:
+    return [m.group(1).strip() for m in _LIST_ITEM.finditer(text)]
+
+
+# ---------------------------------------------------------------------------
+# Numerical values with context
+# ---------------------------------------------------------------------------
+
+_NUMBER_CONTEXT = re.compile(
+    r"(\b\d+(?:\.\d+)?)\s*(%|ms|s|sec|min|hr|hours?|days?|bytes?|[KMGT]B"
+    r"|tokens?|requests?|errors?|users?|GB|MB|KB)\b",
+    re.IGNORECASE,
+)
+
+
+def _extract_numerical(text: str) -> list[str]:
+    """Extract numbers with trailing unit context."""
+    return [f"{m.group(1)} {m.group(2)}" for m in _NUMBER_CONTEXT.finditer(text)]
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+class StatisticalExtractor:
+    """Stage 2 — statistical NLP extraction (no ML models)."""
+
+    def __init__(self, top_k_sentences: int = 5) -> None:
+        self._top_k = top_k_sentences
+
+    def extract(
+        self,
+        text: str,
+        source_window_id: str = "",
+    ) -> list[Fact]:
+        """Extract key sentences, noun phrases, headers, list items, numbers."""
+        facts: list[Fact] = []
+
+        # --- Key sentences (TextRank) ---
+        sentences = _sentences(text)
+        for idx, score in textrank_sentences(sentences, self._top_k):
+            facts.append(Fact(
+                text=sentences[idx],
+                category="key_sentence",
+                source_window_id=source_window_id,
+                confidence=min(0.85, 0.75 + score),
+                extraction_stage=2,
+                metadata={"textrank_score": round(score, 4), "sentence_index": idx},
+            ))
+
+        # --- Noun phrases ---
+        for np_text in _extract_noun_phrases(text):
+            facts.append(Fact(
+                text=np_text,
+                category="noun_phrase",
+                source_window_id=source_window_id,
+                confidence=0.75,
+                extraction_stage=2,
+            ))
+
+        # --- Section headers ---
+        for header in _extract_headers(text):
+            facts.append(Fact(
+                text=header,
+                category="section_header",
+                source_window_id=source_window_id,
+                confidence=0.85,
+                extraction_stage=2,
+            ))
+
+        # --- List items ---
+        for item in _extract_list_items(text):
+            facts.append(Fact(
+                text=item,
+                category="list_item",
+                source_window_id=source_window_id,
+                confidence=0.80,
+                extraction_stage=2,
+            ))
+
+        # --- Numerical values ---
+        for num in _extract_numerical(text):
+            facts.append(Fact(
+                text=num,
+                category="numerical_value",
+                source_window_id=source_window_id,
+                confidence=0.80,
+                extraction_stage=2,
+            ))
+
+        return facts

crp/extraction/stage3_gliner.py

@@ -0,0 +1,210 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Stage 3 — GLiNER zero-shot NER extraction (SHOULD, ~50ms, lazy model load).
+
+Trigger: Stage 2 yield < self-calibrated baseline.
+Model: GLiNER (~200MB), loaded lazily, unloaded after 20 idle windows.
+Graceful fallback: if model unavailable, returns empty and logs warning.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Protocol, runtime_checkable
+
+from crp.extraction.types import Fact
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# GLiNER model protocol (for dependency inversion)
+# ---------------------------------------------------------------------------
+
+@runtime_checkable
+class GLiNERModel(Protocol):
+    """Minimal interface a GLiNER-compatible model must satisfy."""
+
+    def predict_entities(
+        self, text: str, labels: list[str], threshold: float = 0.5
+    ) -> list[dict[str, Any]]:
+        """Return list of dicts with keys: text, label, score, start, end."""
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Label derivation from task context
+# ---------------------------------------------------------------------------
+
+def derive_labels_from_noun_phrases(noun_phrases: list[str], max_labels: int = 15) -> list[str]:
+    """Convert Stage 2 noun phrases into zero-shot NER labels.
+
+    Strips determiners and lowercases. De-duplicates and caps at *max_labels*.
+    """
+    labels: list[str] = []
+    seen: set[str] = set()
+    for np_text in noun_phrases:
+        # Simple normalisation
+        label = np_text.strip().lower()
+        for prefix in ("the ", "a ", "an "):
+            if label.startswith(prefix):
+                label = label[len(prefix):]
+        if label and label not in seen:
+            seen.add(label)
+            labels.append(label)
+        if len(labels) >= max_labels:
+            break
+    return labels
+
+
+# ---------------------------------------------------------------------------
+# Stage 3 Extractor
+# ---------------------------------------------------------------------------
+
+class GLiNERExtractor:
+    """Stage 3 — zero-shot NER via GLiNER (lazy, optional).
+
+    The model is loaded on first call and unloaded after *idle_limit* windows
+    without a call. If ``gliner`` is not installed, all calls return ``[]``.
+    """
+
+    def __init__(self, idle_limit: int = 20) -> None:
+        self._model: GLiNERModel | None = None
+        self._idle_limit = idle_limit
+        self._windows_since_use: int = 0
+        self._available: bool | None = None  # None = not yet probed
+
+    # -- Lifecycle ----------------------------------------------------------
+
+    @staticmethod
+    def _is_model_cached() -> bool:
+        """Check if the GLiNER model is already cached locally.
+
+        Checks the HuggingFace hub cache directory for the model files.
+        Returns True if cached, False if a download would be required.
+        """
+        try:
+            from pathlib import Path
+            # HuggingFace hub default cache location
+            hf_home = os.environ.get("HF_HOME", os.environ.get(
+                "HUGGINGFACE_HUB_CACHE",
+                str(Path.home() / ".cache" / "huggingface" / "hub"),
+            ))
+            cache_dir = Path(hf_home) / "models--urchade--gliner_base"
+            if cache_dir.is_dir():
+                # Check that snapshots exist (actual model files)
+                snapshots = cache_dir / "snapshots"
+                if snapshots.is_dir() and any(snapshots.iterdir()):
+                    return True
+        except Exception:  # noqa: BLE001
+            pass
+        return False
+
+    def _ensure_model(self) -> GLiNERModel | None:
+        """Lazy-load GLiNER. Returns None if unavailable.
+
+        If the model is not cached locally and we appear to be offline,
+        skips the download attempt rather than hanging on a network call.
+        """
+        if self._available is False:
+            return None
+        if self._model is not None:
+            return self._model
+
+        # Check cache before attempting download — if not cached and
+        # offline, avoid a long timeout or network failure.
+        if not self._is_model_cached():
+            # Quick connectivity probe: can we reach huggingface.co?
+            import socket
+            try:
+                socket.create_connection(("huggingface.co", 443), timeout=3)
+            except OSError:
+                self._available = False
+                logger.warning(
+                    "GLiNER model not cached and network unreachable — "
+                    "Stage 3 disabled. Pre-download with: "
+                    'python -c "from gliner import GLiNER; '
+                    "GLiNER.from_pretrained('urchade/gliner_base')\""
+                )
+                return None
+
+        try:
+            from gliner import GLiNER  # type: ignore[import-untyped]
+
+            self._model = GLiNER.from_pretrained("urchade/gliner_base")  # type: ignore[assignment]
+            self._available = True
+            logger.info("GLiNER model loaded successfully")
+            return self._model
+        except Exception:
+            self._available = False
+            logger.warning("GLiNER not available — Stage 3 will be skipped")
+            return None
+
+    def unload(self) -> None:
+        """Release model from memory."""
+        self._model = None
+        logger.info("GLiNER model unloaded (idle)")
+
+    def tick_idle(self) -> None:
+        """Called once per window. Unloads model after idle_limit."""
+        if self._model is not None:
+            self._windows_since_use += 1
+            if self._windows_since_use >= self._idle_limit:
+                self.unload()
+
+    @property
+    def is_available(self) -> bool:
+        if self._available is None:
+            self._ensure_model()
+        return self._available is True
+
+    # -- Extraction ---------------------------------------------------------
+
+    def extract(
+        self,
+        text: str,
+        labels: list[str] | None = None,
+        source_window_id: str = "",
+        threshold: float = 0.5,
+    ) -> list[Fact]:
+        """Run zero-shot NER over *text* using *labels*.
+
+        If no labels provided, uses a small set of generic security/tech labels.
+        Returns ``[]`` if model is unavailable — never raises.
+        """
+        model = self._ensure_model()
+        if model is None:
+            return []
+
+        self._windows_since_use = 0  # Reset idle counter
+
+        if not labels:
+            labels = [
+                "vulnerability", "server", "endpoint", "credential",
+                "configuration", "service", "network", "error",
+                "tool", "technique", "software", "organization",
+            ]
+
+        try:
+            entities = model.predict_entities(text, labels, threshold=threshold)
+        except Exception:
+            logger.exception("GLiNER prediction failed")
+            return []
+
+        facts: list[Fact] = []
+        for ent in entities:
+            confidence = float(ent.get("score", 0.65))
+            facts.append(Fact(
+                text=str(ent.get("text", "")),
+                category=str(ent.get("label", "entity")),
+                source_window_id=source_window_id,
+                confidence=min(0.85, max(0.65, confidence)),
+                extraction_stage=3,
+                metadata={
+                    "label": ent.get("label"),
+                    "gliner_score": round(confidence, 4),
+                    "span": [ent.get("start", 0), ent.get("end", 0)],
+                },
+            ))
+
+        return facts