contextos-memory-runtime 1.0.0rc2__py3-none-any.whl

contextos/services/ingestion.py

@@ -0,0 +1,143 @@
+"""Two-stage privacy-gated candidate ingestion for ContextOS Phase 3."""
+
+from __future__ import annotations
+
+from contextos.core.enums import EventType, PrivacyDecision, SecretDetectionMode
+from contextos.core.exceptions import IngestionError, SecretDetectedError
+from contextos.core.models import IngestRequest, IngestResult, RawEvent
+from contextos.core.protocols import (
+    EmbeddingService,
+    EventRepository,
+    LexicalIndex,
+    MemoryExtractor,
+    MemoryRepository,
+    SecretScanner,
+    TokenCounter,
+    VectorStore,
+)
+from contextos.services.privacy import PrivacyGate
+
+
+class IngestionPipeline:
+    """Sanitize raw input, extract candidates, and gate candidates again.
+
+    Raw input exists only in the caller/request and transient local variables.
+    Only sanitized event content and value-free findings can reach persistence.
+    Later-stage constructor dependencies remain for wiring compatibility and are
+    deliberately not invoked during candidate extraction.
+    """
+
+    def __init__(
+        self,
+        *,
+        secret_scanner: SecretScanner,
+        memory_extractor: MemoryExtractor,
+        memory_repo: MemoryRepository,
+        event_repo: EventRepository,
+        embedding_service: EmbeddingService,
+        vector_store: VectorStore,
+        lexical_index: LexicalIndex,
+        token_counter: TokenCounter,
+        secret_detection_mode: SecretDetectionMode = SecretDetectionMode.STRICT,
+    ) -> None:
+        self._extractor = memory_extractor
+        self._event_repo = event_repo
+        self._secret_mode = secret_detection_mode
+        self._privacy_gate = PrivacyGate(secret_scanner)
+
+    async def ingest(self, request: IngestRequest) -> IngestResult:
+        warnings: list[str] = []
+
+        # skip_secret_scan is retained in the request model for compatibility,
+        # but it cannot bypass the Phase 3 persistence boundary.
+        gated_input = self._privacy_gate.gate_input(
+            request.content,
+            source_type=request.source_type,
+            source_uri=request.source_uri,
+            tags=request.tags,
+            source_role=request.source_role,
+            mode=self._secret_mode,
+        )
+        assessment = gated_input.assessment
+        secrets_detected = assessment.has_findings
+        secrets_redacted = assessment.decision in {
+            PrivacyDecision.REDACT,
+            PrivacyDecision.QUARANTINE,
+        }
+        safe_assessment = assessment.model_dump(exclude={"sanitized_text"})
+
+        if assessment.decision == PrivacyDecision.REJECT:
+            await self._event_repo.append(RawEvent(
+                event_type=EventType.SECRET_DETECTED,
+                source_type=gated_input.source_type,
+                source_uri=gated_input.source_uri,
+                content=None,
+                metadata={"privacy_decision": assessment.decision.value},
+                privacy_scan_result=safe_assessment,
+            ))
+            raise SecretDetectedError(
+                sorted({finding.category.value for finding in assessment.findings}),
+                "Input rejected by the pre-ingest privacy gate.",
+            )
+
+        if len(gated_input.content) > 100_000:
+            raise IngestionError("Input exceeds maximum length of 100000 characters")
+
+        sanitized_content = gated_input.content
+        event = RawEvent(
+            event_type=EventType.INGEST,
+            source_type=gated_input.source_type,
+            source_uri=gated_input.source_uri,
+            content=sanitized_content,
+            metadata={
+                "original_length": len(request.content),
+                "processed_length": len(sanitized_content),
+                "privacy_decision": assessment.decision.value,
+                "scan_bypass_requested": request.skip_secret_scan,
+                "source_role": request.source_role.value,
+                "source_trust": assessment.source_trust.value,
+            },
+            privacy_scan_result=safe_assessment,
+        )
+        await self._event_repo.append(event)
+
+        if assessment.decision == PrivacyDecision.QUARANTINE:
+            warnings.append("Input quarantined by the privacy gate; no candidates extracted.")
+            return IngestResult(
+                event_id=event.id,
+                candidates=[],
+                privacy_assessment=assessment,
+                secrets_detected=secrets_detected,
+                secrets_redacted=True,
+                warnings=warnings,
+            )
+
+        extracted = await self._extractor.extract(
+            text=sanitized_content,
+            source_type=gated_input.source_type,
+            source_uri=gated_input.source_uri,
+            suggested_type=request.memory_type,
+            tags=gated_input.tags,
+            source_role=request.source_role,
+            confirmed_user_information=request.confirmed_user_information,
+        )
+        candidates, blocked = self._privacy_gate.gate_candidates(
+            extracted, source_trust=assessment.source_trust
+        )
+        if secrets_redacted:
+            warnings.append("Sensitive values were redacted before extraction.")
+        if blocked:
+            warnings.append(f"Privacy gate blocked {len(blocked)} candidate(s).")
+        if not candidates:
+            warnings.append("No safe memory candidates could be extracted from the input.")
+
+        # Acceptance, long-term memory persistence, embedding, and indexing are later phases.
+        return IngestResult(
+            event_id=event.id,
+            candidates=candidates,
+            privacy_assessment=assessment,
+            blocked_candidate_assessments=blocked,
+            secrets_detected=secrets_detected,
+            secrets_redacted=secrets_redacted,
+            warnings=warnings,
+        )

contextos/services/inspection.py

@@ -0,0 +1,174 @@
+"""Bounded RAG inspection built from one Phase 13 explanation execution."""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+from uuid import uuid4
+
+from pydantic import BaseModel, Field
+
+from contextos.core.enums import RetrievalMode
+from contextos.core.models import RetrievalQuery
+from contextos.services.explainability import (
+    ExplainabilityService,
+    ExplanationRequest,
+    ExplanationTrace,
+)
+from contextos.services.token_counter import get_token_counter_for_model
+
+
+class InspectionRequest(ExplanationRequest):
+    graph: bool = False
+    compare: bool = False
+    target_model: str | None = Field(default=None, max_length=200)
+
+
+class RAGInspection(BaseModel):
+    inspection_id: str
+    explanation_trace_id: str
+    query: dict[str, Any]
+    configuration: dict[str, Any]
+    stages: list[dict[str, Any]]
+    candidates: list[dict[str, Any]]
+    context_diff: dict[str, Any]
+    final_context: dict[str, Any]
+    requested_memory: dict[str, Any] | None = None
+    comparison: dict[str, Any] | None = None
+    provider_dispatch: dict[str, Any]
+    timing: dict[str, float]
+    content: str | None = None
+
+
+class RAGInspector:
+    def __init__(self, services: dict[str, Any]) -> None:
+        self._services = services
+        self._explain: ExplainabilityService = services["explainability"]
+
+    async def inspect(self, request: InspectionRequest) -> RAGInspection:
+        started = time.perf_counter()
+        trace, retrieved, selection, compiled = await self._explain.execute(request)
+        counter = (
+            get_token_counter_for_model(request.target_model)
+            if request.target_model
+            else self._explain.token_counter
+        )
+        if counter is None:
+            raise RuntimeError("Inspection token counter is unavailable")
+        tokenizer_name = counter.encoding_name
+        if request.target_model and tokenizer_name.startswith("profile-"):
+            tokenizer_name = (
+                "qwen-profile" if "qwen" in request.target_model.lower() else "claude-profile"
+            )
+        candidate_tokens = sum(counter.count(row.memory.content) for row in retrieved.memories)
+        optimized_tokens = sum(
+            counter.count(row.memory.content) for row in selection.selected_memories
+        )
+        compiled_tokens = counter.count(compiled.context_text)
+        avoided = max(0, candidate_tokens - compiled_tokens)
+        stages = [
+            {
+                **stage,
+                "removed_count": max(0, stage["input_count"] - stage["output_count"]),
+            }
+            for stage in trace.stages[:20]
+        ]
+        duplicate_count = sum(fact.reason.value == "duplicate" for fact in compiled.excluded_facts)
+        compiled_provenance = sum(bool(fact.provenance_event_ids) for fact in compiled.facts)
+        comparison = await self._compare(request, trace) if request.compare else None
+        return RAGInspection(
+            inspection_id=str(uuid4()),
+            explanation_trace_id=trace.trace_id,
+            query={"characters": len(request.query), "content_redacted": True},
+            configuration={
+                "mode": trace.strategy,
+                "graph": trace.strategy
+                in {RetrievalMode.GRAPH.value, RetrievalMode.HYBRID_GRAPH.value},
+                "budget": request.budget,
+                "limit": request.limit,
+                "temporal_scope": request.temporal_scope.value,
+                "target_model_requested": request.target_model is not None,
+            },
+            stages=stages,
+            candidates=trace.candidates,
+            context_diff={
+                "candidate_tokens": candidate_tokens,
+                "optimized_tokens": optimized_tokens,
+                "compiled_tokens": compiled_tokens,
+                "tokens_removed": avoided,
+                "net_token_change": compiled_tokens - candidate_tokens,
+                "reduction_ratio": avoided / candidate_tokens if candidate_tokens else 0.0,
+                "selected_memories": len(selection.selected_memories),
+                "rejected_memories": max(
+                    0, len(retrieved.memories) - len(selection.selected_memories)
+                ),
+                "facts_emitted": len(compiled.facts),
+                "facts_excluded": len(compiled.excluded_facts),
+                "duplicate_facts_excluded": duplicate_count,
+                "provenance_coverage": (
+                    compiled_provenance / len(compiled.facts) if compiled.facts else None
+                ),
+                "token_measurement_source": counter.measurement_source.value,
+                "tokenizer": tokenizer_name,
+                "token_basis": "target_model_recount"
+                if request.target_model
+                else "pipeline_counter_recount",
+                "text_exposed": request.include_content,
+            },
+            final_context=trace.final_context,
+            requested_memory=trace.requested_memory,
+            comparison=comparison,
+            provider_dispatch=trace.provider_dispatch,
+            timing={
+                "pipeline_ms": trace.measured_pipeline_ms,
+                "inspection_ms": (time.perf_counter() - started) * 1000,
+            },
+            content=trace.content,
+        )
+
+    async def _compare(self, request: InspectionRequest, trace: ExplanationTrace) -> dict[str, Any]:
+        """Explicit extra retrievals; no optimizer/compiler replay or quality claims."""
+        modes = (
+            RetrievalMode.LEXICAL,
+            RetrievalMode.DENSE,
+            RetrievalMode.HYBRID,
+            RetrievalMode.HYBRID_GRAPH,
+        )
+        primary_ids = [row["memory_id"] for row in trace.candidates]
+        rank_primary = {memory_id: rank for rank, memory_id in enumerate(primary_ids, 1)}
+        rows: list[dict[str, Any]] = []
+        for mode in modes:
+            started = time.perf_counter()
+            result = await self._services["retrieval"].retrieve(
+                RetrievalQuery(
+                    text=request.query,
+                    mode=mode,
+                    k=request.limit,
+                    temporal_scope=request.temporal_scope,
+                )
+            )
+            ids = [str(item.memory.id) for item in result.memories]
+            rows.append(
+                {
+                    "mode": mode.value,
+                    "candidate_ids": ids,
+                    "candidate_count": len(ids),
+                    "overlap_with_inspection": len(set(ids) & set(primary_ids)),
+                    "rank_movement": [
+                        {
+                            "memory_id": memory_id,
+                            "inspection_rank": rank_primary[memory_id],
+                            "comparison_rank": rank,
+                        }
+                        for rank, memory_id in enumerate(ids, 1)
+                        if memory_id in rank_primary
+                    ],
+                    "latency_ms": (time.perf_counter() - started) * 1000,
+                }
+            )
+        return {
+            "runs": rows,
+            "contextos_final_memory_ids": trace.final_context["memories_contributing"],
+            "ground_truth_metrics": None,
+            "quality_status": "NOT_AVAILABLE: no ground truth supplied",
+        }

contextos/services/memory.py

@@ -0,0 +1,291 @@
+"""Memory management service for ContextOS.
+
+Business logic layer for memory CRUD, lifecycle transitions, and search.
+Enforces state machine rules from core.enums.VALID_TRANSITIONS.
+"""
+
+from __future__ import annotations
+
+import logging
+from uuid import UUID
+
+from contextos.core.enums import VALID_TRANSITIONS, EventType, MemoryStatus
+from contextos.core.exceptions import (
+    InvalidTransitionError,
+    MemoryNotFoundError,
+)
+from contextos.core.models import (
+    Memory,
+    MemoryFilters,
+    MemoryUpdate,
+    RawEvent,
+    ScoredMemory,
+)
+from contextos.core.protocols import (
+    EmbeddingService,
+    EventRepository,
+    LexicalIndex,
+    MemoryRepository,
+    VectorStore,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class CoreMemoryService:
+    """Phase 1 memory CRUD and lifecycle without retrieval or indexing dependencies."""
+
+    def __init__(self, repository: MemoryRepository) -> None:
+        self._repository = repository
+
+    async def create(self, memory: Memory) -> Memory:
+        return await self._repository.create(memory)
+
+    async def get(self, memory_id: UUID) -> Memory | None:
+        return await self._repository.get(memory_id)
+
+    async def list(self, filters: MemoryFilters) -> list[Memory]:
+        return await self._repository.list(filters)
+
+    async def update(self, memory_id: UUID, update: MemoryUpdate) -> Memory:
+        current = await self._repository.get(memory_id)
+        if current is None:
+            raise MemoryNotFoundError(str(memory_id))
+        return await self._repository.update(memory_id, update, current.version)
+
+    async def transition(self, memory_id: UUID, status: MemoryStatus) -> Memory:
+        current = await self._repository.get(memory_id)
+        if current is None:
+            raise MemoryNotFoundError(str(memory_id))
+        return await self._repository.update_status(memory_id, status, current.version)
+
+    async def supersede(self, old_id: UUID, successor: Memory) -> Memory:
+        current = await self._repository.get(old_id)
+        if current is None:
+            raise MemoryNotFoundError(str(old_id))
+        return await self._repository.supersede(old_id, successor, current.version)
+
+    async def delete(self, memory_id: UUID) -> Memory:
+        return await self.transition(memory_id, MemoryStatus.DELETED)
+
+
+class MemoryManager:
+    """Business logic for memory management.
+
+    Implements the MemoryService protocol.
+    """
+
+    def __init__(
+        self,
+        *,
+        memory_repo: MemoryRepository,
+        event_repo: EventRepository,
+        vector_store: VectorStore,
+        lexical_index: LexicalIndex,
+        embedding_service: EmbeddingService,
+    ) -> None:
+        self._memory_repo = memory_repo
+        self._event_repo = event_repo
+        self._vector_store = vector_store
+        self._lexical_index = lexical_index
+        self._embedding_service = embedding_service
+
+    async def get(self, memory_id: UUID) -> Memory | None:
+        return await self._memory_repo.get(memory_id)
+
+    async def list(self, filters: MemoryFilters) -> list[Memory]:
+        return await self._memory_repo.list(filters)
+
+    async def search(self, query: str, limit: int = 50) -> list[ScoredMemory]:
+        """Quick search across memories using vector similarity."""
+        try:
+            query_vec = await self._embedding_service.embed_query(query)
+            results = await self._vector_store.search(vector=query_vec, top_k=limit)
+
+            scored: list[ScoredMemory] = []
+            for r in results:
+                memory = await self._memory_repo.get(UUID(r.id))
+                if memory and memory.status == MemoryStatus.ACTIVE:
+                    scored.append(ScoredMemory(
+                        memory=memory,
+                        final_score=r.score,
+                        vector_score=r.score,
+                    ))
+                    await self._memory_repo.update_access(memory.id)
+
+            return scored
+        except Exception:
+            logger.warning("Vector search failed in memory manager", exc_info=True)
+            return []
+
+    async def update(self, memory_id: UUID, update: MemoryUpdate) -> Memory:
+        memory = await self._memory_repo.get(memory_id)
+        if memory is None:
+            raise MemoryNotFoundError(str(memory_id))
+
+        # Record the update event (before/after snapshot)
+        before_content = memory.content
+
+        updated = await self._memory_repo.update(
+            memory_id, update, expected_version=memory.version
+        )
+
+        # If content changed, re-embed and re-index
+        if update.content is not None and update.content != before_content:
+            try:
+                embeddings = await self._embedding_service.embed([updated.content])
+                await self._vector_store.delete([str(memory_id)])
+                await self._vector_store.add(
+                    ids=[str(memory_id)],
+                    vectors=embeddings,
+                    metadata=[{"type": updated.type.value, "status": updated.status.value}],
+                )
+            except Exception:
+                logger.warning("Failed to re-embed memory %s", memory_id, exc_info=True)
+
+            try:
+                await self._lexical_index.delete(str(memory_id))
+                await self._lexical_index.index(
+                    doc_id=str(memory_id),
+                    text=updated.content,
+                    metadata={"type": updated.type.value},
+                )
+            except Exception:
+                logger.warning("Failed to re-index memory %s", memory_id, exc_info=True)
+
+        # Record event
+        await self._event_repo.append(RawEvent(
+            event_type=EventType.MEMORY_UPDATED,
+            source_type="system",
+            metadata={
+                "memory_id": str(memory_id),
+                "before_content": before_content,
+                "after_content": updated.content,
+                "fields_changed": [
+                    k for k, v in update.model_dump(exclude_none=True).items()
+                ],
+            },
+            memory_ids=[memory_id],
+        ))
+
+        return updated
+
+    async def transition(
+        self, memory_id: UUID, new_status: MemoryStatus, reason: str = ""
+    ) -> Memory:
+        """Transition a memory to a new lifecycle state."""
+        memory = await self._memory_repo.get(memory_id)
+        if memory is None:
+            raise MemoryNotFoundError(str(memory_id))
+
+        # Validate transition
+        allowed = VALID_TRANSITIONS.get(memory.status, set())
+        if new_status not in allowed:
+            raise InvalidTransitionError(
+                str(memory_id), memory.status.value, new_status.value
+            )
+
+        updated = await self._memory_repo.update_status(
+            memory_id, new_status, expected_version=memory.version
+        )
+
+        # Record transition event
+        await self._event_repo.append(RawEvent(
+            event_type=EventType.MEMORY_TRANSITION,
+            source_type="system",
+            metadata={
+                "memory_id": str(memory_id),
+                "from_status": memory.status.value,
+                "to_status": new_status.value,
+                "reason": reason,
+            },
+            memory_ids=[memory_id],
+        ))
+
+        logger.info(
+            "Memory %s transitioned: %s → %s (reason: %s)",
+            memory_id, memory.status.value, new_status.value, reason,
+        )
+
+        return updated
+
+    async def delete(self, memory_id: UUID) -> None:
+        """Soft delete: transition to DELETED, remove from indices."""
+        memory = await self._memory_repo.get(memory_id)
+        if memory is None:
+            raise MemoryNotFoundError(str(memory_id))
+
+        # Transition to DELETED (validates allowed transitions)
+        if memory.status != MemoryStatus.DELETED:
+            # Find the path to DELETED
+            allowed = VALID_TRANSITIONS.get(memory.status, set())
+            if MemoryStatus.DELETED not in allowed:
+                raise InvalidTransitionError(
+                    str(memory_id), memory.status.value, MemoryStatus.DELETED.value
+                )
+            await self._memory_repo.update_status(
+                memory_id, MemoryStatus.DELETED, expected_version=memory.version
+            )
+
+        # Remove from indices
+        try:
+            await self._vector_store.delete([str(memory_id)])
+        except Exception:
+            logger.warning("Failed to remove memory %s from vector store", memory_id)
+        try:
+            await self._lexical_index.delete(str(memory_id))
+        except Exception:
+            logger.warning("Failed to remove memory %s from BM25 index", memory_id)
+
+        # Record event
+        await self._event_repo.append(RawEvent(
+            event_type=EventType.MEMORY_DELETED,
+            source_type="system",
+            metadata={"memory_id": str(memory_id), "type": "soft_delete"},
+            memory_ids=[memory_id],
+        ))
+
+    async def purge(self, memory_id: UUID) -> None:
+        """Hard delete: destroy from all stores."""
+        memory = await self._memory_repo.get(memory_id)
+        if memory is None:
+            raise MemoryNotFoundError(str(memory_id))
+
+        # Remove from indices
+        try:
+            await self._vector_store.delete([str(memory_id)])
+        except Exception:
+            logger.warning("Failed to purge memory %s from vector store", memory_id)
+        try:
+            await self._lexical_index.delete(str(memory_id))
+        except Exception:
+            logger.warning("Failed to purge memory %s from BM25 index", memory_id)
+
+        # Remove from database
+        await self._memory_repo.delete(memory_id)
+
+        # Record purge event (no content — it's destroyed)
+        await self._event_repo.append(RawEvent(
+            event_type=EventType.MEMORY_PURGED,
+            source_type="system",
+            metadata={"memory_id": str(memory_id), "type": "hard_delete"},
+        ))
+
+        logger.info("Memory %s purged from all stores", memory_id)
+
+    async def history(self, topic: str) -> list[Memory]:
+        """Get temporal evolution of memories related to a topic.
+
+        Returns memories (including historical/superseded) related to the topic,
+        ordered by creation time.
+        """
+        # Search across all statuses
+        scored = await self.search(topic, limit=100)
+
+        # Also include non-active memories that match
+        all_memories = [sm.memory for sm in scored]
+
+        # Sort by creation time ascending (oldest first)
+        all_memories.sort(key=lambda m: m.created_at)
+
+        return all_memories