crprotocol 2.0.0__py3-none-any.whl

crp/security/quarantine.py

@@ -0,0 +1,249 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Ingest quarantine — anti-poisoning with 1-window quarantine (§7.8).
+
+Facts from untrusted sources are quarantined for 1 window with a 0.7×
+confidence penalty. Cross-reference validation promotes or rejects them.
+Batch poisoning detection: >30% failures → quarantine entire batch.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+
+
+@dataclass
+class QuarantineEntry:
+    """A fact held in quarantine."""
+
+    fact_id: str
+    original_confidence: float
+    penalized_confidence: float  # 0.7× original
+    quarantine_window_id: str
+    ingested_at: float = field(default_factory=time.time)
+    source_label: str = ""
+    promoted: bool = False
+    rejected: bool = False
+    rejection_reason: str = ""
+
+
+@dataclass
+class QuarantineReport:
+    """Result of cross-reference validation pass."""
+
+    total_quarantined: int = 0
+    promoted: int = 0
+    rejected: int = 0
+    batch_poisoned: bool = False
+    details: list[str] = field(default_factory=list)
+
+
+# Confidence penalty factor for quarantined facts (§6F.1)
+QUARANTINE_CONFIDENCE_FACTOR = 0.7
+# Batch poisoning threshold: >30% failures → quarantine entire batch (§6F.3)
+BATCH_POISON_THRESHOLD = 0.30
+
+
+class IngestQuarantine:
+    """1-window quarantine with confidence penalty and batch poisoning detection (§7.8).
+
+    Workflow:
+    1. Incoming facts go into quarantine with 0.7× confidence
+    2. After 1 window, cross-reference against extraction-derived facts
+    3. Matching facts are promoted (confidence restored)
+    4. Non-matching facts are rejected
+    5. If >30% of a batch fails, quarantine entire batch
+
+    Usage:
+        q = IngestQuarantine()
+        q.quarantine_facts([...], "w-1", source="user_input")
+        # ... next window processes ...
+        report = q.validate_and_promote("w-2", extraction_fact_texts)
+    """
+
+    def __init__(
+        self,
+        confidence_factor: float = QUARANTINE_CONFIDENCE_FACTOR,
+        batch_poison_threshold: float = BATCH_POISON_THRESHOLD,
+    ) -> None:
+        self._factor = confidence_factor
+        self._batch_threshold = batch_poison_threshold
+        self._quarantined: dict[str, QuarantineEntry] = {}
+        self._window_batches: dict[str, list[str]] = {}  # window_id → [fact_ids]
+        self._fact_texts: dict[str, str] = {}  # fact_id → original text for similarity
+        self._promotion_history: list[QuarantineReport] = []
+
+    @property
+    def quarantine_count(self) -> int:
+        """Number of facts currently in quarantine (not promoted or rejected)."""
+        return sum(
+            1 for e in self._quarantined.values()
+            if not e.promoted and not e.rejected
+        )
+
+    @property
+    def history(self) -> list[QuarantineReport]:
+        return list(self._promotion_history)
+
+    def quarantine_fact(
+        self,
+        fact_id: str,
+        original_confidence: float,
+        window_id: str,
+        source_label: str = "",
+        fact_text: str = "",
+    ) -> QuarantineEntry:
+        """Place a single fact into quarantine with 0.7× confidence penalty (§6F.1)."""
+        entry = QuarantineEntry(
+            fact_id=fact_id,
+            original_confidence=original_confidence,
+            penalized_confidence=original_confidence * self._factor,
+            quarantine_window_id=window_id,
+            source_label=source_label,
+        )
+        self._quarantined[fact_id] = entry
+        if fact_text:
+            self._fact_texts[fact_id] = fact_text
+        self._window_batches.setdefault(window_id, []).append(fact_id)
+        return entry
+
+    def quarantine_facts(
+        self,
+        facts: list[tuple[str, float]] | list[tuple[str, float, str]],
+        window_id: str,
+        source_label: str = "",
+    ) -> list[QuarantineEntry]:
+        """Quarantine a batch of facts.
+
+        Facts can be (fact_id, confidence) or (fact_id, confidence, text).
+        """
+        entries = []
+        for item in facts:
+            if len(item) >= 3:
+                fid, conf, text = item[0], item[1], item[2]
+            else:
+                fid, conf, text = item[0], item[1], ""
+            entries.append(
+                self.quarantine_fact(fid, conf, window_id, source_label, text)
+            )
+        return entries
+
+    def get_penalized_confidence(self, fact_id: str) -> float | None:
+        """Get quarantine-penalized confidence for a fact."""
+        entry = self._quarantined.get(fact_id)
+        if entry is None:
+            return None
+        if entry.promoted:
+            return entry.original_confidence
+        return entry.penalized_confidence
+
+    def is_quarantined(self, fact_id: str) -> bool:
+        """Check if a fact is in active quarantine."""
+        entry = self._quarantined.get(fact_id)
+        if entry is None:
+            return False
+        return not entry.promoted and not entry.rejected
+
+    def validate_and_promote(
+        self,
+        current_window_id: str,
+        extraction_fact_texts: dict[str, str],
+        similarity_threshold: float = 0.5,
+    ) -> QuarantineReport:
+        """Cross-reference validation: promote or reject quarantined facts (§6F.2).
+
+        Facts quarantined in an earlier window are validated against
+        extraction-derived facts. Text overlap > threshold → promote.
+
+        Args:
+            current_window_id: Current window being processed
+            extraction_fact_texts: {fact_id: text} from extraction pipeline
+            similarity_threshold: Word overlap threshold for cross-reference
+
+        Returns:
+            QuarantineReport with promotion/rejection counts
+        """
+        report = QuarantineReport()
+        extraction_words_sets = {
+            fid: set(text.lower().split())
+            for fid, text in extraction_fact_texts.items()
+        }
+
+        # Check only facts quarantined in a PREVIOUS window
+        pending = [
+            (fid, entry) for fid, entry in self._quarantined.items()
+            if not entry.promoted and not entry.rejected
+            and entry.quarantine_window_id != current_window_id
+        ]
+
+        report.total_quarantined = len(pending)
+        promoted_count = 0
+        rejected_count = 0
+
+        for fid, entry in pending:
+            # Cross-reference against extraction facts via word overlap
+            matched = False
+            # Look up the quarantined fact's text for similarity comparison
+            q_text = self._fact_texts.get(fid, "")
+            q_words = set(q_text.lower().split()) if q_text else set()
+
+            for ext_fid, ext_words in extraction_words_sets.items():
+                # Exact ID match — same fact re-extracted
+                if ext_fid == fid:
+                    matched = True
+                    break
+                # Word-overlap similarity: Jaccard-like measure
+                if q_words and ext_words:
+                    overlap = len(q_words & ext_words)
+                    union = len(q_words | ext_words)
+                    if union > 0 and (overlap / union) >= similarity_threshold:
+                        matched = True
+                        break
+
+            if matched:
+                entry.promoted = True
+                promoted_count += 1
+                report.details.append(f"promoted:{fid}")
+            else:
+                entry.rejected = True
+                entry.rejection_reason = "no_cross_reference"
+                rejected_count += 1
+                report.details.append(f"rejected:{fid}")
+
+        report.promoted = promoted_count
+        report.rejected = rejected_count
+
+        # Batch poisoning detection (§6F.3)
+        if report.total_quarantined > 0:
+            reject_ratio = rejected_count / report.total_quarantined
+            if reject_ratio > self._batch_threshold:
+                report.batch_poisoned = True
+                # Mark all promoted facts as rejected too
+                for _fid, entry in pending:
+                    if entry.promoted:
+                        entry.promoted = False
+                        entry.rejected = True
+                        entry.rejection_reason = "batch_poisoning"
+                        report.promoted -= 1
+                        report.rejected += 1
+                report.details.append(
+                    f"batch_poisoned: {reject_ratio:.0%} > {self._batch_threshold:.0%}"
+                )
+
+        self._promotion_history.append(report)
+        return report
+
+    def get_active_entries(self) -> list[QuarantineEntry]:
+        """Return all currently quarantined (non-promoted, non-rejected) entries."""
+        return [
+            e for e in self._quarantined.values()
+            if not e.promoted and not e.rejected
+        ]
+
+    def clear(self) -> None:
+        """Clear all quarantine state."""
+        self._quarantined.clear()
+        self._window_batches.clear()
+        self._fact_texts.clear()
+        self._promotion_history.clear()

crp/security/rbac.py

@@ -0,0 +1,221 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""RBAC & rate limiting — role-based access control + per-session limits (§7.10).
+
+Roles: OBSERVER (read-only), OPERATOR (dispatch + ingest), ADMIN (full).
+Rate limits: 60 req/min dispatch, 100 MB/min ingest, per-session token cap.
+"""
+
+from __future__ import annotations
+
+import time
+from collections import deque
+from dataclasses import dataclass, field
+from enum import IntEnum
+
+
+class Role(IntEnum):
+    """RBAC roles with increasing privilege (§6G.1)."""
+
+    OBSERVER = 0  # Read-only: inspect state, read facts
+    OPERATOR = 1  # Dispatch + ingest: all observer + dispatch, ingest
+    ADMIN = 2  # Full: all operator + config, delete, export
+
+
+class Permission(str):
+    """Named permission strings."""
+
+    # Observer permissions
+    READ_FACTS = "read_facts"
+    READ_STATE = "read_state"
+    READ_ENVELOPE = "read_envelope"
+    READ_EVENTS = "read_events"
+
+    # Operator permissions
+    DISPATCH = "dispatch"
+    INGEST = "ingest"
+    STORE_FACTS = "store_facts"
+
+    # Admin permissions
+    DELETE_FACTS = "delete_facts"
+    EXPORT_STATE = "export_state"
+    CONFIGURE = "configure"
+    MANAGE_SESSIONS = "manage_sessions"
+
+
+# Role → allowed permissions
+_ROLE_PERMISSIONS: dict[Role, frozenset[str]] = {
+    Role.OBSERVER: frozenset({
+        Permission.READ_FACTS,
+        Permission.READ_STATE,
+        Permission.READ_ENVELOPE,
+        Permission.READ_EVENTS,
+    }),
+    Role.OPERATOR: frozenset({
+        Permission.READ_FACTS,
+        Permission.READ_STATE,
+        Permission.READ_ENVELOPE,
+        Permission.READ_EVENTS,
+        Permission.DISPATCH,
+        Permission.INGEST,
+        Permission.STORE_FACTS,
+        Permission.EXPORT_STATE,
+        Permission.CONFIGURE,
+    }),
+    Role.ADMIN: frozenset({
+        Permission.READ_FACTS,
+        Permission.READ_STATE,
+        Permission.READ_ENVELOPE,
+        Permission.READ_EVENTS,
+        Permission.DISPATCH,
+        Permission.INGEST,
+        Permission.STORE_FACTS,
+        Permission.DELETE_FACTS,
+        Permission.EXPORT_STATE,
+        Permission.CONFIGURE,
+        Permission.MANAGE_SESSIONS,
+    }),
+}
+
+
+@dataclass
+class RateLimitConfig:
+    """Rate limiting configuration (§6G.2)."""
+
+    dispatch_per_minute: int = 60
+    ingest_mb_per_minute: float = 100.0
+    session_token_cap: int = 0  # 0 = unlimited
+
+
+@dataclass
+class RateLimitState:
+    """Sliding-window rate limit tracking."""
+
+    dispatch_timestamps: deque[float] = field(default_factory=deque)
+    ingest_bytes: deque[tuple[float, int]] = field(default_factory=deque)
+    session_tokens_used: int = 0
+
+
+@dataclass
+class AccessResult:
+    """Result of an RBAC check."""
+
+    allowed: bool
+    reason: str = ""
+    role: Role = Role.OBSERVER
+
+
+class RBACEnforcer:
+    """Role-based access control + rate limiting (§7.10).
+
+    Usage:
+        rbac = RBACEnforcer(role=Role.OPERATOR)
+        result = rbac.check_permission("dispatch")
+        if not result.allowed:
+            raise PermissionError(result.reason)
+        result = rbac.check_rate_limit("dispatch")
+        if not result.allowed:
+            raise RateLimitError(result.reason)
+        rbac.record_dispatch()
+    """
+
+    def __init__(
+        self,
+        role: Role = Role.OPERATOR,
+        config: RateLimitConfig | None = None,
+    ) -> None:
+        self._role = role
+        self._config = config or RateLimitConfig()
+        self._limits = RateLimitState()
+        self._permissions = _ROLE_PERMISSIONS[role]
+
+    @property
+    def role(self) -> Role:
+        return self._role
+
+    @role.setter
+    def role(self, value: Role) -> None:
+        self._role = value
+        self._permissions = _ROLE_PERMISSIONS[value]
+
+    # ── Permission checks (§6G.1) ─────────────────────────────
+
+    def check_permission(self, permission: str) -> AccessResult:
+        """Check if current role has the given permission."""
+        if permission in self._permissions:
+            return AccessResult(allowed=True, role=self._role)
+        return AccessResult(
+            allowed=False,
+            reason=f"Role {self._role.name} lacks permission '{permission}'",
+            role=self._role,
+        )
+
+    def has_permission(self, permission: str) -> bool:
+        return permission in self._permissions
+
+    # ── Rate limiting (§6G.2) ─────────────────────────────────
+
+    def check_rate_limit(self, operation: str, payload_bytes: int = 0) -> AccessResult:
+        """Check if the operation is within rate limits."""
+        now = time.time()
+        window_start = now - 60.0  # 1-minute sliding window
+
+        if operation == Permission.DISPATCH:
+            # Clean old entries
+            while self._limits.dispatch_timestamps and self._limits.dispatch_timestamps[0] < window_start:
+                self._limits.dispatch_timestamps.popleft()
+
+            if len(self._limits.dispatch_timestamps) >= self._config.dispatch_per_minute:
+                return AccessResult(
+                    allowed=False,
+                    reason=f"Dispatch rate limit exceeded: {self._config.dispatch_per_minute}/min",
+                    role=self._role,
+                )
+
+        elif operation == Permission.INGEST:
+            # Clean old entries
+            while self._limits.ingest_bytes and self._limits.ingest_bytes[0][0] < window_start:
+                self._limits.ingest_bytes.popleft()
+
+            total_mb = sum(b for _, b in self._limits.ingest_bytes) / (1024 * 1024)
+            incoming_mb = payload_bytes / (1024 * 1024)
+            if total_mb + incoming_mb > self._config.ingest_mb_per_minute:
+                return AccessResult(
+                    allowed=False,
+                    reason=f"Ingest rate limit exceeded: {self._config.ingest_mb_per_minute} MB/min",
+                    role=self._role,
+                )
+
+        # Session token cap
+        if self._config.session_token_cap > 0:
+            if self._limits.session_tokens_used >= self._config.session_token_cap:
+                return AccessResult(
+                    allowed=False,
+                    reason=f"Session token cap reached: {self._config.session_token_cap}",
+                    role=self._role,
+                )
+
+        return AccessResult(allowed=True, role=self._role)
+
+    # ── Recording operations ──────────────────────────────────
+
+    def record_dispatch(self, tokens_used: int = 0) -> None:
+        """Record a dispatch operation for rate limiting."""
+        self._limits.dispatch_timestamps.append(time.time())
+        self._limits.session_tokens_used += tokens_used
+
+    def record_ingest(self, payload_bytes: int) -> None:
+        """Record an ingest operation for rate limiting."""
+        self._limits.ingest_bytes.append((time.time(), payload_bytes))
+
+    def record_tokens(self, count: int) -> None:
+        """Record token consumption."""
+        self._limits.session_tokens_used += count
+
+    @property
+    def session_tokens_used(self) -> int:
+        return self._limits.session_tokens_used
+
+    def reset_limits(self) -> None:
+        """Reset all rate limit counters."""
+        self._limits = RateLimitState()

crp/security/validation.py

@@ -0,0 +1,164 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Input validation — Layer 1 structural validation, CANNOT be disabled (§7.4).
+
+Enforces:
+- Size limit: 50 MB
+- Unicode NFC normalization
+- Null byte stripping
+- Control character stripping (preserves \\n, \\t, \\r)
+- MIME type validation
+- Metadata key count ≤ 50
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class ValidationResult:
+    """Result of input validation."""
+
+    valid: bool
+    sanitized_text: str
+    warnings: list[str] = field(default_factory=list)
+    original_size: int = 0
+    sanitized_size: int = 0
+    null_bytes_removed: int = 0
+    control_chars_removed: int = 0
+    metadata_keys_truncated: int = 0
+
+
+# Allowed MIME types for fact ingestion
+ALLOWED_MIME_TYPES = frozenset({
+    "text/plain",
+    "text/markdown",
+    "text/html",
+    "text/csv",
+    "application/json",
+    "application/xml",
+    "text/xml",
+    "application/yaml",
+    "text/yaml",
+})
+
+# Size limits
+MAX_INPUT_SIZE = 50 * 1024 * 1024  # 50 MB
+MAX_METADATA_KEYS = 50
+
+# Control character regex: match all control chars except \n \t \r
+_CONTROL_CHAR_RE = re.compile(
+    r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]"
+)
+
+
+class InputValidator:
+    """Structural input validation — Layer 1, cannot be disabled (§7.4).
+
+    This validator ALWAYS runs on all input. It cannot be turned off.
+    It performs structural sanitization without modifying semantic content.
+
+    Usage:
+        validator = InputValidator()
+        result = validator.validate("input text")
+        if result.valid:
+            use(result.sanitized_text)
+    """
+
+    def __init__(
+        self,
+        max_size: int = MAX_INPUT_SIZE,
+        max_metadata_keys: int = MAX_METADATA_KEYS,
+    ) -> None:
+        self._max_size = max_size
+        self._max_metadata_keys = max_metadata_keys
+
+    def validate(
+        self,
+        text: str,
+        mime_type: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> ValidationResult:
+        """Validate and sanitize input text (§7.4).
+
+        Steps:
+        1. Size check (50 MB limit)
+        2. Unicode NFC normalization
+        3. Null byte stripping
+        4. Control character stripping (keep \\n, \\t, \\r)
+        5. MIME type validation (if provided)
+        6. Metadata key count check (≤ 50)
+        """
+        warnings: list[str] = []
+        original_size = len(text.encode("utf-8"))
+
+        # 1. Size limit (§6D.2)
+        if original_size > self._max_size:
+            return ValidationResult(
+                valid=False,
+                sanitized_text="",
+                warnings=[f"Input exceeds size limit: {original_size} > {self._max_size} bytes"],
+                original_size=original_size,
+            )
+
+        # 2. Unicode NFC normalization (§6D.2)
+        sanitized = unicodedata.normalize("NFC", text)
+
+        # 3. Null byte stripping (§6D.2)
+        null_count = sanitized.count("\x00")
+        if null_count > 0:
+            sanitized = sanitized.replace("\x00", "")
+            warnings.append(f"Stripped {null_count} null bytes")
+
+        # 4. Control character stripping — keep \n \t \r (§6D.3)
+        control_matches = _CONTROL_CHAR_RE.findall(sanitized)
+        control_count = len(control_matches)
+        if control_count > 0:
+            sanitized = _CONTROL_CHAR_RE.sub("", sanitized)
+            warnings.append(f"Stripped {control_count} control characters")
+
+        # 5. MIME type validation (§6D.4)
+        if mime_type is not None and mime_type not in ALLOWED_MIME_TYPES:
+            warnings.append(f"Unrecognized MIME type: {mime_type}")
+
+        # 6. Metadata key count (§6D.4)
+        metadata_truncated = 0
+        if metadata is not None and len(metadata) > self._max_metadata_keys:
+            metadata_truncated = len(metadata) - self._max_metadata_keys
+            warnings.append(
+                f"Metadata has {len(metadata)} keys, max {self._max_metadata_keys}; "
+                f"excess keys will be ignored"
+            )
+
+        sanitized_size = len(sanitized.encode("utf-8"))
+
+        return ValidationResult(
+            valid=True,
+            sanitized_text=sanitized,
+            warnings=warnings,
+            original_size=original_size,
+            sanitized_size=sanitized_size,
+            null_bytes_removed=null_count,
+            control_chars_removed=control_count,
+            metadata_keys_truncated=metadata_truncated,
+        )
+
+    def validate_metadata(self, metadata: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
+        """Validate and truncate metadata keys (§6D.4).
+
+        Returns (sanitized_metadata, warnings).
+        """
+        warnings: list[str] = []
+        if len(metadata) <= self._max_metadata_keys:
+            return dict(metadata), warnings
+
+        warnings.append(
+            f"Metadata truncated: {len(metadata)} → {self._max_metadata_keys} keys"
+        )
+        # Keep first N keys (deterministic order)
+        truncated = dict(list(metadata.items())[:self._max_metadata_keys])
+        return truncated, warnings

crp/state/__init__.py

@@ -0,0 +1,31 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""State management — facts, warm store, event log, snapshots, cold storage."""
+
+from .cold_storage import PersistedStateHeader, persist_to_cold, restore_from_cold
+from .compaction import CompactionConfig, CompactionResult, compact, should_compact  # noqa: F401
+from .critical_state import CriticalState, StructuralState
+from .event_log import FactEventLog
+from .fact import StateFact, set_embedding_function
+from .serialization import FactGraphSerializer
+from .snapshot import EventLogSnapshot, SnapshotManager
+from .warm_store import WarmStateStore, WarmStoreConfig
+
+__all__ = [
+    "CriticalState",
+    "CompactionConfig",
+    "CompactionResult",
+    "compact",
+    "EventLogSnapshot",
+    "FactEventLog",
+    "FactGraphSerializer",
+    "SnapshotManager",
+    "StateFact",
+    "StructuralState",
+    "WarmStateStore",
+    "WarmStoreConfig",
+    "PersistedStateHeader",
+    "persist_to_cold",
+    "restore_from_cold",
+    "set_embedding_function",
+]