contextos-memory-runtime 1.0.0rc2__py3-none-any.whl

contextos/services/retrieval_index.py

@@ -0,0 +1,88 @@
+"""Explicit synchronization of ephemeral retrieval indexes with SQLite."""
+
+from __future__ import annotations
+
+import asyncio
+
+from contextos.core.enums import MemoryStatus
+from contextos.core.models import Memory, MemoryFilters
+from contextos.core.protocols import EmbeddingService, LexicalIndex, MemoryRepository, VectorStore
+
+
+INDEXED_STATUSES = (
+    MemoryStatus.ACTIVE,
+    MemoryStatus.HISTORICAL,
+    MemoryStatus.SUPERSEDED,
+    MemoryStatus.CONTRADICTED,
+    MemoryStatus.EXPIRED,
+)
+
+
+class RetrievalIndexSynchronizer:
+    """Rebuild both indexes when the persisted memory fingerprint changes."""
+
+    def __init__(
+        self,
+        *,
+        memory_repo: MemoryRepository,
+        lexical_index: LexicalIndex,
+        vector_store: VectorStore,
+        embedding_service: EmbeddingService,
+    ) -> None:
+        self._memory_repo = memory_repo
+        self._lexical_index = lexical_index
+        self._vector_store = vector_store
+        self._embedding_service = embedding_service
+        self._fingerprint: tuple[tuple[str, int, str, str], ...] | None = None
+        self._lock = asyncio.Lock()
+
+    async def ensure_current(self, *, force: bool = False) -> bool:
+        """Synchronize indexes; return whether a rebuild occurred."""
+        async with self._lock:
+            memories = await self._load_indexable_memories()
+            fingerprint = tuple(
+                sorted(
+                    (str(memory.id), memory.version, memory.content_hash, memory.status.value)
+                    for memory in memories
+                )
+            )
+            if not force and fingerprint == self._fingerprint:
+                return False
+            await self._rebuild(memories)
+            self._fingerprint = fingerprint
+            return True
+
+    async def _load_indexable_memories(self) -> list[Memory]:
+        memories: list[Memory] = []
+        for status in INDEXED_STATUSES:
+            offset = 0
+            while True:
+                page = await self._memory_repo.list(
+                    MemoryFilters(status=status, limit=500, offset=offset)
+                )
+                memories.extend(page)
+                if len(page) < 500:
+                    break
+                offset += len(page)
+        return memories
+
+    async def _rebuild(self, memories: list[Memory]) -> None:
+        documents = {str(memory.id): memory.content for memory in memories}
+        ids = [str(memory.id) for memory in memories]
+        vectors = (
+            await self._embedding_service.embed([memory.content for memory in memories])
+            if memories
+            else []
+        )
+        metadata = [
+            {
+                "status": memory.status.value,
+                "type": memory.type.value,
+                "source_type": memory.source_type,
+            }
+            for memory in memories
+        ]
+        # Generate and validate the more failure-prone dense corpus before
+        # replacing either live index.
+        await self._vector_store.rebuild(ids, vectors, metadata)
+        await self._lexical_index.rebuild(documents)

contextos/services/router.py

@@ -0,0 +1,302 @@
+"""Deterministic Model Router for ContextOS."""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+from contextos.core.enums import RoutingPolicy, TokenMeasurementSource
+from contextos.core.exceptions import (
+    ContextWindowExceededError,
+    ModelUnavailableError,
+    ProviderUnavailableError,
+    RoutingFailureError,
+)
+from contextos.core.models import (
+    ModelCapabilities,
+    ModelRequest,
+    RouteDecision,
+)
+from contextos.core.protocols import ModelProvider
+from contextos.services.token_counter import TokenCounter, get_token_counter_for_model
+
+logger = logging.getLogger(__name__)
+
+
+class DeterministicModelRouter:
+    """Selects provider and model deterministically based on policy and constraints."""
+
+    def __init__(
+        self,
+        default_provider_id: str = "fake",
+        default_model_id: str = "fake-default",
+        default_policy: RoutingPolicy = RoutingPolicy.LOCAL_FIRST,
+    ) -> None:
+        self._default_provider_id = default_provider_id
+        self._default_model_id = default_model_id
+        self._default_policy = default_policy
+
+    async def route(
+        self,
+        request: ModelRequest,
+        providers: dict[str, ModelProvider],
+        policy: RoutingPolicy | None = None,
+    ) -> RouteDecision:
+        """Route request to the most appropriate healthy provider and model."""
+        start_time = time.perf_counter()
+        chosen_policy = policy or request.routing_policy or self._default_policy
+
+        if not providers:
+            raise RoutingFailureError(chosen_policy.value, "No providers registered in system")
+
+        candidates_evaluated: list[str] = []
+
+        # -------------------------------------------------------------------
+        # Policy: EXPLICIT
+        # -------------------------------------------------------------------
+        if chosen_policy == RoutingPolicy.EXPLICIT or (request.provider and chosen_policy != RoutingPolicy.FIXED_DEFAULT):
+            prov_id = request.provider or self._default_provider_id
+            if prov_id not in providers:
+                raise ProviderUnavailableError(prov_id, f"Provider '{prov_id}' not found in registry")
+
+            provider = providers[prov_id]
+            is_healthy = await provider.health()
+            if not is_healthy:
+                raise ProviderUnavailableError(prov_id, f"Provider '{prov_id}' is unhealthy")
+
+            models = await provider.list_models()
+            target_model_id = request.model or (models[0].model_id if models else self._default_model_id)
+            candidates_evaluated.append(f"{prov_id}/{target_model_id}")
+
+            matched_model = next((m for m in models if m.model_id == target_model_id), None)
+            if not matched_model:
+                raise ModelUnavailableError(target_model_id, prov_id)
+
+            self._validate_context_window(request, matched_model)
+
+            elapsed_ms = (time.perf_counter() - start_time) * 1000.0
+            return RouteDecision(
+                policy=chosen_policy,
+                reason="Explicit provider/model selection requested and verified",
+                candidates_evaluated=candidates_evaluated,
+                selected_provider=prov_id,
+                selected_model=target_model_id,
+                fallback_used=False,
+                routing_latency_ms=elapsed_ms,
+            )
+
+        # -------------------------------------------------------------------
+        # Policy: FIXED_DEFAULT
+        # -------------------------------------------------------------------
+        if chosen_policy == RoutingPolicy.FIXED_DEFAULT:
+            prov_id = self._default_provider_id
+            if prov_id not in providers:
+                raise ProviderUnavailableError(prov_id, "Default provider not registered")
+
+            provider = providers[prov_id]
+            if not await provider.health():
+                raise ProviderUnavailableError(prov_id, "Default provider is unhealthy")
+
+            models = await provider.list_models()
+            matched_model = next((m for m in models if m.model_id == self._default_model_id), None)
+            if not matched_model and models:
+                matched_model = models[0]
+            if not matched_model:
+                raise ModelUnavailableError(self._default_model_id, prov_id)
+
+            candidates_evaluated.append(f"{prov_id}/{matched_model.model_id}")
+            self._validate_context_window(request, matched_model)
+
+            elapsed_ms = (time.perf_counter() - start_time) * 1000.0
+            return RouteDecision(
+                policy=chosen_policy,
+                reason="Fixed default policy selected configured default model",
+                candidates_evaluated=candidates_evaluated,
+                selected_provider=prov_id,
+                selected_model=matched_model.model_id,
+                fallback_used=False,
+                routing_latency_ms=elapsed_ms,
+            )
+
+        # -------------------------------------------------------------------
+        # Policy: LOCAL_FIRST
+        # -------------------------------------------------------------------
+        if chosen_policy == RoutingPolicy.LOCAL_FIRST:
+            local_providers = [p for p in providers.values() if p.is_local]
+            selected_local: tuple[ModelProvider, ModelCapabilities] | None = None
+            initial_provider_tried: str | None = None
+
+            for prov in local_providers:
+                initial_provider_tried = prov.provider_id
+                try:
+                    if not await prov.health():
+                        candidates_evaluated.append(f"{prov.provider_id} (unhealthy)")
+                        continue
+                    prov_models = await prov.list_models()
+                except Exception:
+                    candidates_evaluated.append(f"{prov.provider_id} (health check failed)")
+                    continue
+
+                for model in prov_models:
+                    candidates_evaluated.append(f"{prov.provider_id}/{model.model_id}")
+                    if self._fits_context_and_capabilities(request, model):
+                        selected_local = (prov, model)
+                        break
+                if selected_local:
+                    break
+
+            if selected_local:
+                prov, model = selected_local
+                elapsed_ms = (time.perf_counter() - start_time) * 1000.0
+                return RouteDecision(
+                    policy=chosen_policy,
+                    reason="Local-first policy found healthy local provider and model fitting context window",
+                    candidates_evaluated=candidates_evaluated,
+                    selected_provider=prov.provider_id,
+                    selected_model=model.model_id,
+                    fallback_used=False,
+                    routing_latency_ms=elapsed_ms,
+                )
+
+            # Local provider not found or unhealthy
+            if not request.allow_fallback:
+                raise ProviderUnavailableError(
+                    initial_provider_tried or "local",
+                    "Local providers unavailable and fallback not allowed by request policy",
+                )
+
+            # Fallback to remote provider
+            remote_providers = [p for p in providers.values() if not p.is_local]
+            selected_remote: tuple[ModelProvider, ModelCapabilities] | None = None
+            for prov in remote_providers:
+                if not await prov.health():
+                    candidates_evaluated.append(f"{prov.provider_id} (unhealthy)")
+                    continue
+                prov_models = await prov.list_models()
+                for model in prov_models:
+                    candidates_evaluated.append(f"{prov.provider_id}/{model.model_id}")
+                    if self._fits_context_and_capabilities(request, model):
+                        selected_remote = (prov, model)
+                        break
+                if selected_remote:
+                    break
+
+            if selected_remote:
+                prov, model = selected_remote
+                elapsed_ms = (time.perf_counter() - start_time) * 1000.0
+                return RouteDecision(
+                    policy=chosen_policy,
+                    reason="Local provider unavailable; successfully fell back to remote provider",
+                    candidates_evaluated=candidates_evaluated,
+                    selected_provider=prov.provider_id,
+                    selected_model=model.model_id,
+                    fallback_used=True,
+                    initial_provider=initial_provider_tried,
+                    fallback_reason="Local provider unavailable or failed health check",
+                    routing_latency_ms=elapsed_ms,
+                )
+
+            raise RoutingFailureError(
+                chosen_policy.value,
+                "Neither local nor remote fallback providers could satisfy the request",
+            )
+
+        # -------------------------------------------------------------------
+        # Policy: CAPABILITY_AWARE
+        # -------------------------------------------------------------------
+        if chosen_policy == RoutingPolicy.CAPABILITY_AWARE:
+            all_candidates: list[tuple[ModelProvider, ModelCapabilities]] = []
+            for prov in providers.values():
+                if not await prov.health():
+                    continue
+                models = await prov.list_models()
+                for model in models:
+                    candidates_evaluated.append(f"{prov.provider_id}/{model.model_id}")
+                    if self._fits_context_and_capabilities(request, model):
+                        all_candidates.append((prov, model))
+
+            if not all_candidates:
+                raise RoutingFailureError(
+                    chosen_policy.value,
+                    f"No model satisfied required capabilities: {request.required_capabilities}",
+                )
+
+            # Prefer local if available, then by largest context window
+            all_candidates.sort(key=lambda item: (not item[0].is_local, -item[1].context_window))
+            best_prov, best_model = all_candidates[0]
+
+            elapsed_ms = (time.perf_counter() - start_time) * 1000.0
+            return RouteDecision(
+                policy=chosen_policy,
+                reason="Capability-aware policy matched required capabilities and context fit",
+                candidates_evaluated=candidates_evaluated,
+                selected_provider=best_prov.provider_id,
+                selected_model=best_model.model_id,
+                fallback_used=False,
+                routing_latency_ms=elapsed_ms,
+            )
+
+        raise RoutingFailureError(str(chosen_policy), f"Unsupported routing policy '{chosen_policy}'")
+
+    def _compute_safety_margin(self, counter: TokenCounter) -> int:
+        """Conservative buffer for chat framing (roles, formatting) and approximation variance."""
+        base_framing = 16
+        if counter.measurement_source == TokenMeasurementSource.APPROXIMATED:
+            return base_framing + 32
+        return base_framing
+
+    def _fits_context_and_capabilities(
+        self, request: ModelRequest, model: ModelCapabilities
+    ) -> bool:
+        """Check capability constraints and context window fit."""
+        if not model.enabled:
+            return False
+
+        # Capability checks
+        if "tools" in request.required_capabilities and not model.supports_tools:
+            return False
+        if "json" in request.required_capabilities and not model.supports_json:
+            return False
+        if "vision" in request.required_capabilities and not model.supports_vision:
+            return False
+
+        # Context window fit with safety margin for framing/approximation
+        context_text = request.compiled_context.context_text if request.compiled_context else ""
+        counter = get_token_counter_for_model(model.model_id, model.tokenizer_family)
+        prompt_tokens = counter.count(request.user_prompt)
+        if request.system_prompt:
+            prompt_tokens += counter.count(request.system_prompt)
+        context_tokens = counter.count(context_text) if context_text else 0
+        reserved_output = request.max_output_tokens or 1024
+        margin = self._compute_safety_margin(counter)
+
+        total_needed = prompt_tokens + context_tokens + reserved_output + margin
+        return total_needed <= model.context_window
+
+    def _validate_context_window(
+        self, request: ModelRequest, model: ModelCapabilities
+    ) -> None:
+        """Ensure total needed tokens fit within the model context window.
+
+        Raises ContextWindowExceededError if it does not fit.
+        """
+        context_text = request.compiled_context.context_text if request.compiled_context else ""
+        counter = get_token_counter_for_model(model.model_id, model.tokenizer_family)
+        prompt_tokens = counter.count(request.user_prompt)
+        if request.system_prompt:
+            prompt_tokens += counter.count(request.system_prompt)
+        context_tokens = counter.count(context_text) if context_text else 0
+        reserved_output = request.max_output_tokens or 1024
+        margin = self._compute_safety_margin(counter)
+
+        total_needed = prompt_tokens + context_tokens + reserved_output + margin
+        if total_needed > model.context_window:
+            raise ContextWindowExceededError(
+                model_id=model.model_id,
+                required_tokens=total_needed,
+                context_window=model.context_window,
+                prompt_tokens=prompt_tokens,
+                compiled_context_tokens=context_tokens,
+                reserved_output_tokens=reserved_output,
+            )

contextos/services/secret_scanner.py

@@ -0,0 +1,207 @@
+"""Local structured credential detection and overlap-safe redaction."""
+
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass
+
+from contextos.core.enums import SecretType
+from contextos.core.models import ScanResult, SecretMatch
+
+
+@dataclass(frozen=True, slots=True)
+class SecretPattern:
+    secret_type: SecretType
+    pattern: re.Pattern[str]
+    confidence: float
+    detector: str
+    secret_group: int = 0
+
+
+def _pattern(
+    secret_type: SecretType,
+    expression: str,
+    confidence: float,
+    detector: str,
+    *,
+    flags: int = 0,
+    secret_group: int = 0,
+) -> SecretPattern:
+    return SecretPattern(
+        secret_type, re.compile(expression, flags), confidence, detector, secret_group
+    )
+
+
+SECRET_PATTERNS: list[SecretPattern] = [
+    _pattern(SecretType.PRIVATE_KEY,
+             r"-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----[\s\S]*?"
+             r"-----END (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----",
+             0.99, "private-key-block", flags=re.MULTILINE),
+    _pattern(SecretType.AUTHORIZATION_HEADER,
+             r"\bauthorization\s*:\s*(?:bearer|basic)\s+([^\s,;]+)",
+             0.99, "authorization-header", flags=re.I, secret_group=1),
+    _pattern(SecretType.AWS_ACCESS_KEY, r"(?<![A-Z0-9])AKIA[0-9A-Z]{16}(?![A-Z0-9])",
+             0.98, "aws-access-key"),
+    _pattern(SecretType.GITHUB_TOKEN, r"(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36,255}",
+             0.98, "github-token"),
+    _pattern(SecretType.OPENAI_API_KEY,
+             r"sk-[A-Za-z0-9_-]{20,}T3BlbkFJ[A-Za-z0-9_-]{20,}",
+             0.99, "openai-legacy-key"),
+    _pattern(SecretType.OPENAI_API_KEY, r"sk-proj-[A-Za-z0-9_-]{40,}",
+             0.98, "openai-project-key"),
+    _pattern(SecretType.ANTHROPIC_API_KEY, r"sk-ant-[A-Za-z0-9_-]{40,}",
+             0.98, "anthropic-key"),
+    _pattern(SecretType.GOOGLE_API_KEY, r"AIza[0-9A-Za-z_-]{35}",
+             0.96, "google-api-key"),
+    _pattern(SecretType.SLACK_TOKEN,
+             r"xox[baprs]-[0-9a-zA-Z]{10,}(?:-[0-9a-zA-Z]{10,})*",
+             0.96, "slack-token"),
+    _pattern(SecretType.STRIPE_KEY, r"(?:sk|pk)_(?:test|live)_[0-9a-zA-Z]{24,}",
+             0.98, "stripe-key"),
+    _pattern(SecretType.JWT,
+             r"eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}",
+             0.96, "jwt-token"),
+    _pattern(SecretType.CONNECTION_STRING,
+             r"(?:mongodb|postgres|postgresql|mysql|redis|amqp)(?:\+[a-z]+)?://"
+             r"[^\s:/]+:[^\s@]+@[^\s]+",
+             0.96, "credential-connection-string", flags=re.I),
+    _pattern(SecretType.CONNECTION_STRING,
+             r"(?:https?|ftp)://[^\s:/]+:[^\s@]+@[^\s/]+(?:/[^\s]*)?",
+             0.94, "credential-url-userinfo", flags=re.I),
+    _pattern(SecretType.BEARER_TOKEN, r"\bbearer\s+([A-Za-z0-9._~+/=-]{12,})",
+             0.94, "bearer-token", flags=re.I, secret_group=1),
+    _pattern(SecretType.GENERIC_API_KEY,
+             r"\b(?:api[_ -]?key|apikey|api[_ -]?secret)\b\s*(?:is|=|:)\s*"
+             r"['\"]?([A-Za-z0-9_./+=-]{11,}[A-Za-z0-9_+=-])['\"]?",
+             0.92, "credential-assignment", flags=re.I, secret_group=1),
+    _pattern(SecretType.ACCESS_TOKEN,
+             r"\b(?:access[_ -]?token|refresh[_ -]?token)\b\s*(?:is|=|:)\s*"
+             r"['\"]?([A-Za-z0-9_./+=-]{12,})['\"]?",
+             0.93, "token-assignment", flags=re.I, secret_group=1),
+    _pattern(SecretType.ACCESS_TOKEN,
+             r"[?&](?:x-amz-signature|x-goog-signature|signature|sig)="
+             r"([^&#\s]{16,})",
+             0.95, "signed-url-signature", flags=re.I, secret_group=1),
+    _pattern(SecretType.SESSION_COOKIE,
+             r"\b(?:session(?:[_ -]?(?:id|token))?|cookie)\b\s*(?:is|=|:)\s*"
+             r"['\"]?([A-Za-z0-9_./+=-]{12,})['\"]?",
+             0.91, "session-cookie", flags=re.I, secret_group=1),
+    _pattern(SecretType.PASSWORD,
+             r"\b(?:password|passwd|pwd|pass)\b\s*(?:is|=|:)\s*"
+             r"['\"]?([^\s'\"]{6,})['\"]?",
+             0.91, "password-assignment", flags=re.I, secret_group=1),
+    _pattern(SecretType.OTP,
+             r"\b(?:otp|one[ -]?time (?:password|code)|verification code|pin)\b"
+             r"\s*(?:is|=|:)\s*([0-9]{4,8})\b",
+             0.95, "contextual-otp", flags=re.I, secret_group=1),
+    _pattern(SecretType.AWS_SECRET_KEY,
+             r"\baws[_ -]?secret(?:[_ -]?access)?[_ -]?key\b\s*(?:is|=|:)\s*"
+             r"['\"]?([A-Za-z0-9/+=]{40})['\"]?",
+             0.96, "aws-secret-key", flags=re.I, secret_group=1),
+]
+
+
+def _shannon_entropy(value: str) -> float:
+    if not value:
+        return 0.0
+    frequencies: dict[str, int] = {}
+    for character in value:
+        frequencies[character] = frequencies.get(character, 0) + 1
+    length = len(value)
+    return -sum(
+        (count / length) * math.log2(count / length) for count in frequencies.values()
+    )
+
+
+def _find_high_entropy_strings(
+    text: str,
+    min_length: int = 20,
+    max_length: int = 200,
+    entropy_threshold: float = 4.5,
+) -> list[SecretMatch]:
+    """Optional advisory detector; disabled by default to control false positives."""
+    matches: list[SecretMatch] = []
+    token_pattern = re.compile(
+        r"[A-Za-z0-9+/=_-]{" + str(min_length) + "," + str(max_length) + "}"
+    )
+    for match in token_pattern.finditer(text):
+        entropy = _shannon_entropy(match.group())
+        if entropy >= entropy_threshold:
+            matches.append(SecretMatch(
+                secret_type=SecretType.HIGH_ENTROPY,
+                start=match.start(),
+                end=match.end(),
+                matched_text="[REDACTED:high_entropy]",
+                confidence=min(0.89, (entropy - entropy_threshold) / 2.0 + 0.5),
+            ))
+    return matches
+
+
+class PatternSecretScanner:
+    """Replaceable local scanner implementing the existing SecretScanner protocol."""
+
+    def __init__(
+        self,
+        *,
+        min_confidence: float = 0.5,
+        enable_entropy: bool = False,
+        entropy_threshold: float = 4.5,
+    ) -> None:
+        self._min_confidence = min_confidence
+        self._enable_entropy = enable_entropy
+        self._entropy_threshold = entropy_threshold
+
+    def scan(self, text: str) -> ScanResult:
+        if not text:
+            return ScanResult(scanned_length=0)
+        matches: list[SecretMatch] = []
+        for secret_pattern in SECRET_PATTERNS:
+            if secret_pattern.confidence < self._min_confidence:
+                continue
+            for match in secret_pattern.pattern.finditer(text):
+                start, end = match.span(secret_pattern.secret_group)
+                matches.append(SecretMatch(
+                    secret_type=secret_pattern.secret_type,
+                    start=start,
+                    end=end,
+                    matched_text=f"[REDACTED:{secret_pattern.secret_type.value}]",
+                    confidence=secret_pattern.confidence,
+                ))
+        if self._enable_entropy:
+            matches.extend(_find_high_entropy_strings(
+                text, entropy_threshold=self._entropy_threshold
+            ))
+        matches = self._deduplicate_overlapping(matches)
+        return ScanResult(
+            has_secrets=bool(matches), matches=matches, scanned_length=len(text)
+        )
+
+    def redact(self, text: str) -> tuple[str, ScanResult]:
+        result = self.scan(text)
+        redacted = text
+        for match in sorted(result.matches, key=lambda item: item.start, reverse=True):
+            redacted = (
+                redacted[:match.start]
+                + f"[REDACTED:{match.secret_type.value}]"
+                + redacted[match.end:]
+            )
+        return redacted, result
+
+    @staticmethod
+    def _deduplicate_overlapping(matches: list[SecretMatch]) -> list[SecretMatch]:
+        selected: list[SecretMatch] = []
+        for current in sorted(matches, key=lambda item: (item.start, -item.confidence, -item.end)):
+            if selected and current.start < selected[-1].end:
+                previous = selected[-1]
+                winner = current if current.confidence > previous.confidence else previous
+                selected[-1] = SecretMatch(
+                    secret_type=winner.secret_type,
+                    start=min(previous.start, current.start),
+                    end=max(previous.end, current.end),
+                    matched_text=f"[REDACTED:{winner.secret_type.value}]",
+                    confidence=max(previous.confidence, current.confidence),
+                )
+                continue
+            selected.append(current)
+        return selected