contextos-memory-runtime 1.0.0rc2__py3-none-any.whl

contextos/daemon/wiring.py

@@ -0,0 +1,296 @@
+"""Dependency wiring for ContextOS daemon.
+
+This is where all protocol implementations are bound to their interfaces.
+One function, called at startup, that assembles the entire dependency graph.
+No magic. No DI framework.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from contextos.config.settings import Settings
+from contextos.core.enums import SecretDetectionMode
+from contextos.storage.database import Database
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from contextos.services.token_counter import TokenCounter
+
+
+async def wire_services(settings: Settings) -> dict[str, Any]:
+    """Create and wire all service instances.
+
+    Returns a dict of {service_name: instance} ready for injection into
+    the API server and CLI.
+    """
+    from contextos.connectors.local_files import LocalFileConnector
+    from contextos.connectors.json_import import JsonImportConnector
+
+    configured_connectors = []
+    for connector_id, roots in settings.connectors.local_files.items():
+        if not roots or any(not root.is_dir() for root in roots):
+            raise ValueError(f"Connector {connector_id} has an invalid local root")
+        configured_connectors.append(LocalFileConnector(connector_id, roots))
+    for connector_id, path in settings.connectors.json_imports.items():
+        if (str(path).startswith(("\\\\", "//")) or
+                str(path.resolve()).startswith(("\\\\", "//")) or not path.is_file()):
+            raise ValueError(f"Connector {connector_id} has an invalid JSON import path")
+        configured_connectors.append(JsonImportConnector(connector_id, path))
+    if len({item.connector_id for item in configured_connectors}) != len(configured_connectors):
+        raise ValueError("Connector IDs must be unique")
+
+    # --- Database ---
+    data_dir = settings.daemon.data_dir
+    data_dir.mkdir(parents=True, exist_ok=True)
+
+    db = Database(data_dir / "contextos.db")
+    try:
+        await db.initialize()
+        return _wire_initialized_services(settings, db, configured_connectors)
+    except BaseException:
+        # Startup can fail before the caller receives services and owns cleanup.
+        await db.close()
+        raise
+
+
+def _wire_initialized_services(
+    settings: Settings, db: Database, configured_connectors: list[Any],
+) -> dict[str, Any]:
+    """Construct services only after the database has a cleanup owner."""
+    services: dict[str, Any] = {"settings": settings}
+    services["database"] = db
+
+    conn = db.connection()
+
+    # --- Repositories ---
+    from contextos.storage.memory_repo import SqliteMemoryRepository
+    from contextos.storage.event_repo import SqliteEventRepository
+
+    memory_repo = SqliteMemoryRepository(conn)
+    event_repo = SqliteEventRepository(conn)
+    from contextos.storage.relation_repo import SqliteRelationRepository
+    from contextos.storage.graph_repo import SqliteGraphRepository
+    relation_repo = SqliteRelationRepository(conn)
+    graph_repo = SqliteGraphRepository(conn)
+    services["memory_repo"] = memory_repo
+    services["event_repo"] = event_repo
+    services["relation_repo"] = relation_repo
+    services["graph_repo"] = graph_repo
+
+    # --- Temporal resolution ---
+    from contextos.services.temporal import TemporalMemoryService
+
+    temporal = TemporalMemoryService(memory_repo)
+    services["temporal"] = temporal
+
+    from contextos.services.graph import MemoryGraphService
+
+    graph = MemoryGraphService(
+        memory_repo=memory_repo,
+        relation_repo=relation_repo,
+        graph_repo=graph_repo,
+    )
+    services["graph"] = graph
+
+    # --- Token Counter ---
+    from contextos.services.token_counter import DeterministicWordTokenCounter, TiktokenCounter
+
+    token_counter: TokenCounter
+    if settings.token_counter.encoding == "deterministic":
+        token_counter = DeterministicWordTokenCounter()
+    else:
+        token_counter = TiktokenCounter(settings.token_counter.encoding)
+    services["token_counter"] = token_counter
+
+    # --- Token-aware optimizer ---
+    from contextos.services.optimization import MemoryContextOptimizer
+
+    optimizer = MemoryContextOptimizer(token_counter=token_counter)
+    services["optimizer"] = optimizer
+
+    # --- Embedding Service ---
+    if settings.embedding.model == "deterministic":
+        # Explicit local/test configuration. This avoids a model download while
+        # retaining the normal retrieval, indexing, graph, and SQLite services.
+        from contextos.embedding.deterministic import DeterministicEmbedding
+        embedding_service = DeterministicEmbedding(16)
+        vector_dimension = 16
+    else:
+        from contextos.embedding.sentence_transformers import SentenceTransformerEmbedding
+        embedding_service = SentenceTransformerEmbedding(
+            model_name=settings.embedding.model,
+            device=settings.embedding.device,
+        )
+        vector_dimension = 384
+    services["embedding"] = embedding_service
+
+    # --- Vector Store ---
+    from contextos.storage.vector.in_memory import InMemoryVectorStore
+
+    vector_store = InMemoryVectorStore(dimension=vector_dimension)
+    services["vector_store"] = vector_store
+
+    # --- BM25 Index ---
+    from contextos.storage.lexical.bm25 import BM25Index
+
+    bm25_index = BM25Index()
+    services["bm25_index"] = bm25_index
+    services["lexical_index"] = bm25_index
+
+    # --- Explicit retrieval index synchronization ---
+    from contextos.services.retrieval_index import RetrievalIndexSynchronizer
+
+    retrieval_index = RetrievalIndexSynchronizer(
+        memory_repo=memory_repo,
+        embedding_service=embedding_service,
+        vector_store=vector_store,
+        lexical_index=bm25_index,
+    )
+    services["retrieval_index"] = retrieval_index
+
+    # --- Secret Scanner ---
+    from contextos.services.secret_scanner import PatternSecretScanner
+
+    scanner = PatternSecretScanner(
+        entropy_threshold=settings.privacy.entropy_threshold,
+    )
+    services["secret_scanner"] = scanner
+
+    # --- Memory Extractor ---
+    from contextos.services.extraction import RuleBasedMemoryExtractor
+
+    extractor = RuleBasedMemoryExtractor()
+    services["memory_extractor"] = extractor
+
+    # --- Ingestion Pipeline ---
+    from contextos.services.ingestion import IngestionPipeline
+
+    secret_mode = SecretDetectionMode(settings.privacy.secret_detection)
+    ingestion = IngestionPipeline(
+        secret_scanner=scanner,
+        memory_extractor=extractor,
+        memory_repo=memory_repo,
+        event_repo=event_repo,
+        embedding_service=embedding_service,
+        vector_store=vector_store,
+        lexical_index=bm25_index,
+        token_counter=token_counter,
+        secret_detection_mode=secret_mode,
+    )
+    services["ingestion"] = ingestion
+
+    # --- Phase 11: connector state and bounded sync manager ---
+    from contextos.storage.connector_repo import SqliteConnectorRepository
+    from contextos.connectors.manager import ConnectorManager
+    connector_repo = SqliteConnectorRepository(conn)
+    services["connector_repo"] = connector_repo
+    services["connectors"] = ConnectorManager(
+        state_repo=connector_repo, ingestion=ingestion, temporal=temporal,
+    )
+    for connector in configured_connectors:
+        services["connectors"].register(connector)
+
+    # --- Memory Manager ---
+    from contextos.services.memory import MemoryManager
+
+    memory_manager = MemoryManager(
+        memory_repo=memory_repo,
+        event_repo=event_repo,
+        vector_store=vector_store,
+        lexical_index=bm25_index,
+        embedding_service=embedding_service,
+    )
+    services["memory"] = memory_manager
+
+    # --- Retrieval Engine ---
+    from contextos.services.retrieval import HybridRetrievalEngine
+
+    base_retrieval = HybridRetrievalEngine(
+        memory_repo=memory_repo,
+        vector_store=vector_store,
+        lexical_index=bm25_index,
+        embedding_service=embedding_service,
+        index_synchronizer=retrieval_index,
+    )
+    from contextos.services.graph_retrieval import GraphAugmentedRetrievalEngine
+
+    retrieval = GraphAugmentedRetrievalEngine(
+        base_engine=base_retrieval,
+        graph_service=graph,
+        memory_repo=memory_repo,
+    )
+    services["base_retrieval"] = base_retrieval
+    services["retrieval"] = retrieval
+
+    # --- Context Compiler ---
+    from contextos.services.compilation import QueryAwareContextCompiler
+
+    compiler = QueryAwareContextCompiler(token_counter=token_counter)
+    services["compilation"] = compiler
+
+    # --- Phase 9: Telemetry Repository & Query Service ---
+    from contextos.storage.telemetry_repo import SqliteTelemetryRepository
+    from contextos.services.telemetry_query import TelemetryQueryService
+
+    telemetry_repo = SqliteTelemetryRepository(conn)
+    telemetry_query = TelemetryQueryService(telemetry_repo)
+    services["telemetry_repo"] = telemetry_repo
+    services["telemetry_query"] = telemetry_query
+
+    # --- Phase 9: Provider Adapters ---
+    from contextos.providers.fake import DeterministicFakeProvider
+    from contextos.providers.ollama import OllamaProvider
+    from contextos.providers.openai_compatible import OpenAICompatibleProvider
+
+    fake_provider = DeterministicFakeProvider()
+    ollama_provider = OllamaProvider()
+    openai_compatible_provider = OpenAICompatibleProvider()
+
+    providers = {
+        fake_provider.provider_id: fake_provider,
+        ollama_provider.provider_id: ollama_provider,
+        openai_compatible_provider.provider_id: openai_compatible_provider,
+    }
+    services["fake_provider"] = fake_provider
+    services["ollama_provider"] = ollama_provider
+    services["openai_compatible_provider"] = openai_compatible_provider
+    services["providers"] = providers
+
+    # --- Phase 9: Model Router ---
+    from contextos.core.enums import RoutingPolicy
+    from contextos.services.router import DeterministicModelRouter
+
+    router = DeterministicModelRouter(
+        default_provider_id="fake",
+        default_model_id="fake-default",
+        default_policy=RoutingPolicy.LOCAL_FIRST,
+    )
+    services["router"] = router
+
+    # --- Phase 9: Unified ContextOS Model Service ---
+    from contextos.services.model_service import ContextOSModelService
+
+    model_service = ContextOSModelService(
+        retrieval_service=retrieval,
+        optimizer=optimizer,
+        compilation_service=compiler,
+        router=router,
+        providers=providers,
+        telemetry_repo=telemetry_repo,
+        token_counter=token_counter,
+    )
+    services["model_service"] = model_service
+
+    from contextos.services.explainability import ExplainabilityService
+    explainability = ExplainabilityService(services)
+    services["explainability"] = explainability
+    model_service.set_explainability_service(explainability)
+    from contextos.services.inspection import RAGInspector
+    services["inspector"] = RAGInspector(services)
+
+    logger.info("All services wired successfully")
+    return services

contextos/demo.py

@@ -0,0 +1,217 @@
+"""Reproducible offline ContextOS walkthrough on a temporary database."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import tempfile
+from pathlib import Path
+from typing import Any
+from uuid import UUID
+
+from httpx import ASGITransport, AsyncClient
+
+import contextos.api.server as api_server
+from contextos.config.settings import DaemonConfig, EmbeddingConfig, Settings, TokenCounterConfig
+from contextos.connectors.fake import FakeConnector
+from contextos.connectors.models import ConnectorItem
+from contextos.core.enums import SourceRole
+from contextos.core.exceptions import SecretDetectedError
+from contextos.core.models import IngestRequest
+from contextos.daemon.wiring import wire_services
+from contextos.services.explainability import ExplanationRequest
+from contextos.services.inspection import InspectionRequest
+
+
+async def run_demo() -> dict[str, Any]:
+    """Exercise real services; synthetic fixture values are discarded on exit."""
+    with tempfile.TemporaryDirectory(prefix="contextos-demo-") as folder:
+        services = await wire_services(
+            Settings(
+                daemon=DaemonConfig(data_dir=Path(folder)),
+                embedding=EmbeddingConfig(model="deterministic"),
+                token_counter=TokenCounterConfig(encoding="deterministic"),
+            )
+        )
+        try:
+
+            async def remember(text: str) -> list[str]:
+                ingested = await services["ingestion"].ingest(
+                    IngestRequest(
+                        content=text,
+                        source_type="manual",
+                        source_role=SourceRole.USER,
+                    )
+                )
+                accepted = [
+                    await services["temporal"].accept(
+                        candidate, provenance_event_id=ingested.event_id
+                    )
+                    for candidate in ingested.candidates
+                ]
+                return [str(item.memory.id) for item in accepted]
+
+            initial = await remember("I prefer concise answers for technical questions.")
+            changed = await remember("I now prefer detailed answers for technical questions.")
+            graph_ids = await remember("My project Atlas uses Ollama for local inference.")
+            connector = FakeConnector(
+                "demo_notes",
+                [
+                    ConnectorItem(
+                        external_id="note-1",
+                        source_type="fake",
+                        source_uri="fake://demo/note-1",
+                        revision="1",
+                        content="I prefer concise technical documentation.",
+                    )
+                ],
+            )
+            services["connectors"].register(connector)
+            first_sync = await services["connectors"].sync("demo_notes")
+            second_sync = await services["connectors"].sync("demo_notes")
+            await services["retrieval_index"].ensure_current()
+            graph_nodes, graph_edges, _ = await services["graph"].rebuild()
+            query = "detailed answers for technical questions"
+            explained = await services["explainability"].explain(
+                ExplanationRequest(
+                    query=query,
+                    graph=False,
+                )
+            )
+            inspected = await services["inspector"].inspect(
+                InspectionRequest(
+                    query=query,
+                    graph=False,
+                    target_memory_id=UUID(changed[0]) if changed else None,
+                )
+            )
+            graph_expansion = await services["graph"].expand(
+                query_text="Atlas",
+                seed_memory_ids=[UUID(graph_ids[0])] if graph_ids else [],
+            )
+            graph_memory = (
+                await services["memory_repo"].get(UUID(graph_ids[0])) if graph_ids else None
+            )
+            asked = await services["model_service"].ask(
+                query, target_provider="fake", target_model="fake-default", explain=True
+            )
+            rejected = False
+            try:
+                await services["ingestion"].ingest(
+                    IngestRequest(
+                        content="My test API key is sk-demoPrivateFixture1234567890",
+                        source_type="manual",
+                        source_role=SourceRole.USER,
+                    )
+                )
+            except SecretDetectedError:
+                rejected = True
+            telemetry = await services["telemetry_query"].summary_range(
+                provider_id="fake", model_id="fake-default"
+            )
+            old_memory = await services["memory_repo"].get(UUID(initial[0])) if initial else None
+            new_memory = await services["memory_repo"].get(UUID(changed[0])) if changed else None
+            dashboard_services = api_server._services.copy()
+            try:
+                api_server.set_services(services)
+                async with AsyncClient(
+                    transport=ASGITransport(app=api_server.create_app()),
+                    base_url="http://localhost",
+                ) as client:
+                    dashboard_response = await client.get("/api/v1/dashboard")
+                    dashboard_response.raise_for_status()
+                    dashboard_data = dashboard_response.json()
+            finally:
+                api_server._services.clear()
+                api_server._services.update(dashboard_services)
+            if not (
+                initial and changed and graph_ids and first_sync.accepted > 0
+                and second_sync.unchanged > 0 and graph_edges > 0
+                and graph_expansion.candidate_scores and explained.trace_id
+                and inspected.candidates and inspected.context_diff["facts_emitted"] > 0
+                and asked.compiled_context.included_memory_ids
+                and asked.compiled_context.total_tokens > 0
+                and asked.telemetry.context_tokens_avoided > 0
+                and asked.telemetry.selected_memory_count > 0
+                and asked.telemetry.compiled_fact_count > 0
+                and asked.compiled_context.provenance_coverage > 0
+            ):
+                raise RuntimeError("Offline demo did not produce all required pipeline evidence")
+            if asked.telemetry.provider_id != "fake" or telemetry.total_invocations != 1:
+                raise RuntimeError(
+                    "Offline demo did not invoke and record FakeProvider exactly once"
+                )
+            demo_connector = next(
+                (item for item in dashboard_data["connectors"] if item["id"] == "demo_notes"),
+                None,
+            )
+            if (
+                dashboard_data["memories"]["active"] < 1
+                or demo_connector is None
+                or demo_connector["tracked_items"] < 1
+                or not dashboard_data["recent"]
+            ):
+                raise RuntimeError(
+                    "Offline demo dashboard is missing memory, connector, or telemetry data"
+                )
+            return {
+                "label": "LOCAL SYNTHETIC OFFLINE DEMO",
+                "token_measurement_source": services["token_counter"].measurement_source.value,
+                "tokenizer": services["token_counter"].encoding_name,
+                "initial_memory_ids": initial[:10],
+                "changed_memory_ids": changed[:10],
+                "temporal": {
+                    "previous_status": old_memory.status.value if old_memory else None,
+                    "current_status": new_memory.status.value if new_memory else None,
+                },
+                "connector_first": {"accepted": first_sync.accepted, "status": first_sync.status},
+                "connector_second": {
+                    "unchanged": second_sync.unchanged,
+                    "status": second_sync.status,
+                },
+                "graph": {
+                    "nodes": graph_nodes,
+                    "edges": graph_edges,
+                    "expansion_candidates": len(graph_expansion.candidate_scores),
+                    "memory_status": graph_memory.status.value if graph_memory else None,
+                    "seed_count": len(graph_expansion.seed_node_ids),
+                    "visited": len(graph_expansion.visited_node_ids),
+                },
+                "explanation": {
+                    "trace_id": explained.trace_id,
+                    "provider_state": explained.provider_dispatch["state"],
+                },
+                "inspection": {
+                    "inspection_id": inspected.inspection_id,
+                    "candidates": len(inspected.candidates),
+                    "context_diff": inspected.context_diff,
+                },
+                "model": {
+                    "provider": asked.route_decision.selected_provider,
+                    "model": asked.route_decision.selected_model,
+                    "dispatch_state": asked.dispatch_evidence.state.value,
+                    "compiled_memory_ids": len(asked.compiled_context.included_memory_ids),
+                    "compiled_tokens": asked.compiled_context.total_tokens,
+                    "tokens_avoided": asked.telemetry.context_tokens_avoided,
+                    "provenance_coverage": asked.compiled_context.provenance_coverage,
+                },
+                "telemetry_invocations": telemetry.total_invocations,
+                "dashboard": {
+                    "active_memories": dashboard_data["memories"]["active"],
+                    "connector_tracked_items": demo_connector["tracked_items"],
+                    "recent_invocations": len(dashboard_data["recent"]),
+                    "graph_projection_dirty": dashboard_data["graph"]["dirty"],
+                },
+                "privacy_secret_rejected": rejected,
+                "ephemeral": True,
+            }
+        finally:
+            await services["database"].close()
+
+
+def main() -> None:
+    print(json.dumps(asyncio.run(run_demo()), indent=2))
+
+
+if __name__ == "__main__":
+    main()

contextos/embedding/__init__.py

@@ -0,0 +1 @@
+"""Embedding package for ContextOS."""

contextos/embedding/deterministic.py

@@ -0,0 +1,76 @@
+"""Small deterministic embedding adapter for offline tests and benchmarks."""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import re
+
+
+_SYNONYMS = {
+    "ai": "model",
+    "artificial": "model",
+    "inference": "runtime",
+    "engine": "runtime",
+    "programming": "language",
+    "coding": "language",
+    "formerly": "previous",
+    "prior": "previous",
+    "before": "previous",
+    "style": "response",
+    "brief": "concise",
+    "short": "concise",
+    "large": "larger",
+    "ram": "memory",
+    "resources": "memory",
+    "crashed": "failed",
+}
+_STOP_WORDS = {
+    "a", "an", "and", "did", "do", "does", "for", "has", "have", "i", "in",
+    "is", "it", "of", "on", "the", "to", "user", "was", "what", "when", "which",
+    "with",
+}
+
+
+class DeterministicEmbedding:
+    """Hashed bag-of-concepts embedding with no model or network dependency."""
+
+    def __init__(self, dimension: int = 128) -> None:
+        if dimension < 8:
+            raise ValueError("dimension must be at least 8")
+        self._dimension = dimension
+
+    def _vectorize(self, text: str) -> list[float]:
+        vector = [0.0] * self._dimension
+        for token in re.findall(r"[a-z0-9]+(?:[+#._-][a-z0-9]+)*", text.casefold()):
+            if token in _STOP_WORDS:
+                continue
+            if token in {"focused", "focuses", "focusing"}:
+                token = "focus"
+            elif (
+                token != "focus"
+                and token.endswith("s")
+                and len(token) > 4
+                and not token.endswith("ss")
+            ):
+                token = token[:-1]
+            token = _SYNONYMS.get(token, token)
+            digest = hashlib.sha256(token.encode("utf-8")).digest()
+            index = int.from_bytes(digest[:4], "big") % self._dimension
+            vector[index] += 1.0
+        norm = math.sqrt(sum(value * value for value in vector))
+        return [value / norm for value in vector] if norm else vector
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        return [self._vectorize(text) for text in texts]
+
+    async def embed_query(self, query: str) -> list[float]:
+        return self._vectorize(query)
+
+    @property
+    def dimension(self) -> int:
+        return self._dimension
+
+    @property
+    def model_name(self) -> str:
+        return "deterministic-hashed-concepts"

contextos/embedding/sentence_transformers.py

@@ -0,0 +1,80 @@
+"""Sentence-transformers embedding service for ContextOS.
+
+Wraps sentence-transformers for local embedding generation.
+Default model: all-MiniLM-L6-v2 (384 dimensions, ~22M parameters, fast).
+"""
+
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SentenceTransformerEmbedding:
+    """Local embedding using sentence-transformers.
+
+    Implements the EmbeddingService protocol.
+    Lazy-loads the model on first use to avoid blocking startup.
+    """
+
+    def __init__(
+        self,
+        model_name: str = "all-MiniLM-L6-v2",
+        device: str = "cpu",
+    ) -> None:
+        self._model_name = model_name
+        self._device = device
+        self._model = None
+        self._dimension: int | None = None
+
+    def _load_model(self) -> None:
+        """Lazily load the model."""
+        if self._model is not None:
+            return
+
+        try:
+            from sentence_transformers import SentenceTransformer
+
+            logger.info("Loading embedding model: %s (device: %s)", self._model_name, self._device)
+            self._model = SentenceTransformer(self._model_name, device=self._device)
+            self._dimension = self._model.get_sentence_embedding_dimension()
+            logger.info(
+                "Embedding model loaded: %s (dim=%d)", self._model_name, self._dimension
+            )
+        except ImportError:
+            raise RuntimeError(
+                "sentence-transformers is required for SentenceTransformerEmbedding. "
+                "Install ContextOS with the embeddings extra: pip install 'contextos-memory-runtime[embeddings]'"
+            )
+        except Exception as e:
+            raise RuntimeError(f"Failed to load embedding model '{self._model_name}': {e}") from e
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        """Embed a batch of texts."""
+        self._load_model()
+        assert self._model is not None
+
+        embeddings = self._model.encode(
+            texts,
+            normalize_embeddings=True,
+            show_progress_bar=False,
+        )
+
+        return embeddings.tolist()
+
+    async def embed_query(self, query: str) -> list[float]:
+        """Embed a single query."""
+        results = await self.embed([query])
+        return results[0]
+
+    @property
+    def dimension(self) -> int:
+        """Embedding vector dimension."""
+        self._load_model()
+        assert self._dimension is not None
+        return self._dimension
+
+    @property
+    def model_name(self) -> str:
+        return self._model_name

contextos/mcp/__init__.py

@@ -0,0 +1,5 @@
+"""MCP transport adapter for ContextOS."""
+
+from contextos.mcp.server import create_mcp_server
+
+__all__ = ["create_mcp_server"]