contextos-memory-runtime 1.0.0rc2__py3-none-any.whl

contextos/benchmarks/mcp.py

@@ -0,0 +1,324 @@
+"""Deterministic local MCP adapter measurement helpers.
+
+Callers provide the already-wired deterministic services used by tests; this
+module never creates a provider or sends network traffic.
+
+Label: LOCAL DEVELOPMENT MACHINE SYNTHETIC INFRASTRUCTURE BENCHMARK
+"""
+
+from __future__ import annotations
+
+import json
+import statistics
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+from contextos.core.models import RetrievalQuery
+from contextos.mcp.server import ContextOSMCPApplication, MCPPermissions
+
+
+@dataclass(frozen=True)
+class OperationResult:
+    """Timing and size for a single benchmark operation."""
+    operation: str
+    iterations: int
+    mean_ms: float
+    median_ms: float
+    p95_ms: float | None  # Only with sufficient samples (>= 20)
+    serialized_bytes: int
+    # Operation-specific metrics
+    selected_memories: int | None = None
+    compiled_facts: int | None = None
+    compiled_tokens: int | None = None
+    graph_nodes: int | None = None
+    privacy_decision: str | None = None
+
+
+@dataclass(frozen=True)
+class MCPBenchmarkReport:
+    """Complete Phase 10 benchmark report."""
+    label: str = "LOCAL DEVELOPMENT MACHINE SYNTHETIC INFRASTRUCTURE BENCHMARK"
+    direct_results: list[OperationResult] = field(default_factory=list)
+    mcp_results: list[OperationResult] = field(default_factory=list)
+    stdio_results: list[OperationResult] = field(default_factory=list)
+    overheads: dict[str, float] = field(default_factory=dict)
+
+
+def _p95(samples: list[float]) -> float | None:
+    if len(samples) < 20:
+        return None
+    return sorted(samples)[int(len(samples) * 0.95)]
+
+
+def _operation_result(
+    operation: str, samples: list[float], response: dict[str, object],
+) -> OperationResult:
+    serialized = len(json.dumps(response, sort_keys=True, default=str).encode("utf-8"))
+    return OperationResult(
+        operation=operation,
+        iterations=len(samples),
+        mean_ms=statistics.mean(samples),
+        median_ms=statistics.median(samples),
+        p95_ms=_p95(samples),
+        serialized_bytes=serialized,
+        selected_memories=int(response.get("selected_memory_count", 0)) if "selected_memory_count" in response else (int(response.get("result_count", 0)) if response.get("ok") else None),
+        compiled_facts=int(response.get("compiled_fact_count", 0)) if "compiled_fact_count" in response else None,
+        compiled_tokens=int(response.get("token_count", 0)) if "token_count" in response else None,
+        graph_nodes=int(response.get("graph_node_count", 0)) if "graph_node_count" in response else None,
+        privacy_decision=str(response.get("error_code")) if response.get("error_code") == "PRIVACY_REJECTED" else None,
+    )
+
+
+async def _measure(coro_factory, iterations: int) -> tuple[list[float], Any]:
+    """Run a coroutine factory N times and collect timings."""
+    samples: list[float] = []
+    last_result = None
+    for _ in range(iterations):
+        started = time.perf_counter()
+        last_result = await coro_factory()
+        elapsed = (time.perf_counter() - started) * 1000
+        samples.append(elapsed)
+    return samples, last_result
+
+
+async def run_phase10_mcp_benchmark(
+    services: dict[str, Any],
+    query: str = "ContextOS memory",
+    iterations: int = 5,
+) -> MCPBenchmarkReport:
+    """Measure direct retrieval versus the local adapter using identical services.
+
+    Exercises: search, compile, graph, remember success, remember privacy
+    rejection, and temporal lookup.
+    """
+    app = ContextOSMCPApplication(services, MCPPermissions(allow_write=True))
+    direct_results: list[OperationResult] = []
+    mcp_results: list[OperationResult] = []
+    overheads: dict[str, float] = {}
+
+    # --- Direct service measurements ---
+
+    # Direct search
+    samples, last = await _measure(
+        lambda: services["retrieval"].retrieve(RetrievalQuery(text=query, k=5)),
+        iterations,
+    )
+    direct_results.append(OperationResult(
+        operation="search", iterations=len(samples),
+        mean_ms=statistics.mean(samples), median_ms=statistics.median(samples),
+        p95_ms=_p95(samples), serialized_bytes=0,
+        selected_memories=len(last.memories) if last else 0,
+    ))
+
+    # --- MCP adapter measurements ---
+
+    # MCP search
+    samples, last = await _measure(
+        lambda: app.invoke("contextos_search_memory", None,
+                           lambda: app.search(query, 5, "hybrid", False)),
+        iterations,
+    )
+    mcp_results.append(_operation_result("search", samples, last))
+    overheads["search_adapter_ms"] = max(0.0, mcp_results[-1].mean_ms - direct_results[0].mean_ms)
+
+    # MCP compile
+    samples, last = await _measure(
+        lambda: app.invoke("contextos_compile_context", None,
+                           lambda: app.compile(query, 1000, "hybrid")),
+        iterations,
+    )
+    mcp_results.append(_operation_result("compile", samples, last))
+
+    # MCP graph
+    samples, last = await _measure(
+        lambda: app.invoke("contextos_graph_neighbors", None,
+                           lambda: app.graph_neighbors(query, 1, 50, 100)),
+        iterations,
+    )
+    mcp_results.append(_operation_result("graph", samples, last))
+
+    # MCP remember (success)
+    samples, last = await _measure(
+        lambda: app.invoke("contextos_remember", None,
+                           lambda: app.remember(f"Benchmark memory about {query}")),
+        iterations,
+    )
+    mcp_results.append(_operation_result("remember_success", samples, last))
+
+    # MCP temporal lookup
+    samples, last = await _measure(
+        lambda: app.invoke("contextos_current_state", None,
+                           lambda: app.current_state("programming_language", "user", "global")),
+        iterations,
+    )
+    mcp_results.append(_operation_result("temporal_lookup", samples, last))
+
+    # MCP telemetry
+    samples, last = await _measure(
+        lambda: app.invoke("contextos_telemetry_summary", None,
+                           app.telemetry_summary),
+        iterations,
+    )
+    mcp_results.append(_operation_result("telemetry", samples, last))
+
+    return MCPBenchmarkReport(
+        direct_results=direct_results,
+        mcp_results=mcp_results,
+        overheads=overheads,
+    )
+
+
+async def main() -> None:
+    """Run the canonical Phase 10 MCP benchmark against a temporary SQLite DB."""
+    import asyncio
+    import tempfile
+    from pathlib import Path
+    from mcp import Client
+    from contextos.core.enums import SecretDetectionMode
+    from contextos.embedding.deterministic import DeterministicEmbedding
+    from contextos.mcp.server import MCPPermissions, create_mcp_server
+    from contextos.services.compilation import QueryAwareContextCompiler
+    from contextos.services.extraction import RuleBasedMemoryExtractor
+    from contextos.services.graph import MemoryGraphService
+    from contextos.services.graph_retrieval import GraphAugmentedRetrievalEngine
+    from contextos.services.ingestion import IngestionPipeline
+    from contextos.services.optimization import MemoryContextOptimizer
+    from contextos.services.retrieval import HybridRetrievalEngine
+    from contextos.services.retrieval_index import RetrievalIndexSynchronizer
+    from contextos.services.secret_scanner import PatternSecretScanner
+    from contextos.services.telemetry_query import TelemetryQueryService
+    from contextos.services.temporal import TemporalMemoryService
+    from contextos.services.token_counter import DeterministicWordTokenCounter
+    from contextos.storage.database import Database
+    from contextos.storage.event_repo import SqliteEventRepository
+    from contextos.storage.graph_repo import SqliteGraphRepository
+    from contextos.storage.lexical.bm25 import BM25Index
+    from contextos.storage.memory_repo import SqliteMemoryRepository
+    from contextos.storage.relation_repo import SqliteRelationRepository
+    from contextos.storage.telemetry_repo import SqliteTelemetryRepository
+    from contextos.storage.vector.in_memory import InMemoryVectorStore
+
+    tmp_dir = Path(tempfile.mkdtemp())
+    db_path = tmp_dir / "bench.db"
+    db = Database(db_path)
+    await db.initialize()
+    conn = db.connection()
+    memory_repo = SqliteMemoryRepository(conn)
+    event_repo = SqliteEventRepository(conn)
+    relation_repo = SqliteRelationRepository(conn)
+    graph_repo = SqliteGraphRepository(conn)
+    telemetry_repo = SqliteTelemetryRepository(conn)
+    embedding = DeterministicEmbedding(16)
+    lexical, vector = BM25Index(), InMemoryVectorStore(16)
+    index = RetrievalIndexSynchronizer(
+        memory_repo=memory_repo, embedding_service=embedding,
+        vector_store=vector, lexical_index=lexical,
+    )
+    graph = MemoryGraphService(
+        memory_repo=memory_repo, relation_repo=relation_repo,
+        graph_repo=graph_repo,
+    )
+    retrieval = GraphAugmentedRetrievalEngine(
+        base_engine=HybridRetrievalEngine(
+            memory_repo=memory_repo, vector_store=vector,
+            lexical_index=lexical, embedding_service=embedding,
+            index_synchronizer=index,
+        ),
+        graph_service=graph, memory_repo=memory_repo,
+    )
+    token_counter = DeterministicWordTokenCounter()
+    temporal = TemporalMemoryService(memory_repo)
+    ingestion = IngestionPipeline(
+        secret_scanner=PatternSecretScanner(),
+        memory_extractor=RuleBasedMemoryExtractor(),
+        memory_repo=memory_repo, event_repo=event_repo,
+        embedding_service=embedding, vector_store=vector,
+        lexical_index=lexical, token_counter=token_counter,
+        secret_detection_mode=SecretDetectionMode.STRICT,
+    )
+    services = {
+        "database": db,
+        "retrieval": retrieval,
+        "optimizer": MemoryContextOptimizer(token_counter=token_counter),
+        "compilation": QueryAwareContextCompiler(token_counter=token_counter),
+        "graph": graph,
+        "temporal": temporal,
+        "ingestion": ingestion,
+        "telemetry_query": TelemetryQueryService(telemetry_repo),
+    }
+
+    # Seed data using MCP adapter
+    server = create_mcp_server(services, MCPPermissions(allow_write=True))
+    async with Client(server) as client:
+        seed_data = [
+            "I am working on Project Atlas. Project Atlas uses Python for machine learning.",
+            "I am working on Project Beta. Project Beta uses Rust for systems programming.",
+            "I currently use Ollama for local inference.",
+            "I am working on Project Atlas. Project Atlas uses llama.cpp.",
+            "I prefer concise answers for technical questions.",
+            "I use Docker for containerization.",
+            "I am working on Project Gamma. Project Gamma uses PostgreSQL.",
+            "I am learning TypeScript for web development.",
+        ]
+        for text in seed_data:
+            await client.call_tool("contextos_remember", {"text": text})
+
+    # Run benchmark
+    report = await run_phase10_mcp_benchmark(services, query="Atlas Python machine learning", iterations=5)
+
+    # Print results
+    print("=" * 70)
+    print("LOCAL DEVELOPMENT MACHINE SYNTHETIC INFRASTRUCTURE BENCHMARK")
+    print("=" * 70)
+    print()
+    print("DIRECT SERVICE RESULTS:")
+    print("-" * 50)
+    for r in report.direct_results:
+        print(f"  {r.operation}:")
+        print(f"    iterations:       {r.iterations}")
+        print(f"    mean latency:     {r.mean_ms:.3f} ms")
+        print(f"    median latency:   {r.median_ms:.3f} ms")
+        if r.p95_ms is not None:
+            print(f"    p95 latency:      {r.p95_ms:.3f} ms")
+        print(f"    serialized bytes: {r.serialized_bytes}")
+        if r.selected_memories is not None:
+            print(f"    selected memories: {r.selected_memories}")
+        print()
+
+    print("MCP ADAPTER RESULTS:")
+    print("-" * 50)
+    for r in report.mcp_results:
+        print(f"  {r.operation}:")
+        print(f"    iterations:       {r.iterations}")
+        print(f"    mean latency:     {r.mean_ms:.3f} ms")
+        print(f"    median latency:   {r.median_ms:.3f} ms")
+        if r.p95_ms is not None:
+            print(f"    p95 latency:      {r.p95_ms:.3f} ms")
+        print(f"    serialized bytes: {r.serialized_bytes}")
+        if r.selected_memories is not None:
+            print(f"    selected memories: {r.selected_memories}")
+        if r.compiled_facts is not None:
+            print(f"    compiled facts:    {r.compiled_facts}")
+        if r.compiled_tokens is not None:
+            print(f"    compiled tokens:   {r.compiled_tokens}")
+        if r.graph_nodes is not None:
+            print(f"    graph nodes:       {r.graph_nodes}")
+        if r.privacy_decision is not None:
+            print(f"    privacy decision:  {r.privacy_decision}")
+        print()
+
+    print("OVERHEAD CALCULATIONS:")
+    print("-" * 50)
+    for key, value in report.overheads.items():
+        print(f"  {key}: {value:.3f} ms")
+    print()
+
+    await db.close()
+    print("Benchmark complete.")
+
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
+

contextos/benchmarks/model_routing.py

@@ -0,0 +1,203 @@
+"""Deterministic Phase 9 benchmark: Model routing, multi-model token comparison, and telemetry overhead."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from uuid import uuid4
+
+from contextos.core.enums import (
+    MemoryStatus,
+    ModelFinishReason,
+    RoutingPolicy,
+    TokenMeasurementSource,
+)
+from contextos.core.models import (
+    CompilationConfig,
+    ContextBudget,
+    Memory,
+    ModelCapabilities,
+    ModelInvocationTelemetry,
+    ModelRequest,
+    ScoredMemory,
+)
+from contextos.providers.fake import DeterministicFakeProvider
+from contextos.services.compilation import QueryAwareContextCompiler
+from contextos.services.optimization import MemoryContextOptimizer
+from contextos.services.router import DeterministicModelRouter
+from contextos.services.token_counter import (
+    ClaudeProfileTokenCounter,
+    QwenProfileTokenCounter,
+    TiktokenCounter,
+    get_token_counter_for_model,
+    recount_cross_model,
+)
+from contextos.storage.database import Database
+from contextos.storage.telemetry_repo import SqliteTelemetryRepository
+
+
+@dataclass(frozen=True)
+class ModelProfileResult:
+    profile_name: str
+    target_token_estimate: int
+    measurement_source: str
+    reduction_ratio: float
+    context_tokens_avoided: int
+
+
+@dataclass(frozen=True)
+class Phase9BenchmarkReport:
+    candidate_context_tokens: int
+    compiled_context_tokens: int
+    profiles: dict[str, ModelProfileResult]
+    router_overhead_ms: float
+    telemetry_overhead_ms: float
+    token_counting_overhead_ms: float
+
+
+def benchmark_memories() -> list[ScoredMemory]:
+    """Diverse benchmark corpus containing code, technical preferences, and project facts."""
+    texts = [
+        "User currently uses Ollama for local model inference with llama3.2.",
+        "User prefers concise technical responses with typed Python 3.12+ syntax and pydantic models.",
+        "Project Atlas architecture uses SQLite with WAL mode, graph projection, and reciprocal rank fusion.",
+        "User's machine is an M2 Max with 64GB unified memory running macOS Sonoma.",
+        "User previously attempted running Qwen-72B locally, but inference failed due to high memory pressure.",
+        "Development setup: pytest for testing, ruff for linting, and mypy in strict mode.",
+        "ContextOS compilation pipeline achieves token reduction while preserving exact fact attribution.",
+        "The local LLM endpoint is hosted at http://127.0.0.1:11434 with zero external telemetry transmission.",
+    ]
+    memories = [
+        Memory(
+            id=uuid4(),
+            content=text,
+            status=MemoryStatus.ACTIVE,
+            token_count=len(text.split()),
+        )
+        for text in texts
+    ]
+    return [
+        ScoredMemory(
+            memory=m,
+            final_score=0.95 - (i * 0.05),
+            retrieval_sources=["lexical", "dense"] if i % 2 == 0 else ["lexical", "graph"],
+        )
+        for i, m in enumerate(memories)
+    ]
+
+
+async def run_phase9_benchmark(db_path: Path | None = None) -> Phase9BenchmarkReport:
+    """Run deterministic Phase 9 benchmark and return structured metrics."""
+    ref_counter = TiktokenCounter("cl100k_base")
+    optimizer = MemoryContextOptimizer(token_counter=ref_counter)
+    compiler = QueryAwareContextCompiler(token_counter=ref_counter)
+
+    candidates = benchmark_memories()
+    candidate_tokens = sum(ref_counter.count(c.memory.content) for c in candidates)
+
+    # 1. Optimize and Compile
+    query = "What is the recommended local model and hardware configuration for Project Atlas?"
+    selection = optimizer.optimize(
+        query=query,
+        candidates=candidates,
+        budget=ContextBudget(max_tokens=2000),
+    )
+    compiled = await compiler.compile(
+        query=query,
+        memories=selection,
+        config=CompilationConfig(budget=1500),
+    )
+    compiled_tokens = compiled.total_tokens
+
+    # 2. Multi-Model Token Recounting (Profiles)
+    tc_start = time.perf_counter()
+    recounts = recount_cross_model(
+        compiled.context_text,
+        ["FAKE_CLAUDE_LIKE", "FAKE_OPENAI_LIKE", "FAKE_QWEN_LIKE"],
+    )
+    tc_overhead = (time.perf_counter() - tc_start) * 1000.0
+
+    profiles: dict[str, ModelProfileResult] = {}
+    for prof_name, (count, src) in recounts.items():
+        # Context tokens avoided from perspective of this model profile
+        prof_counter = get_token_counter_for_model(prof_name, prof_name)
+        cand_for_model = sum(prof_counter.count(c.memory.content) for c in candidates)
+        avoided = max(0, cand_for_model - count)
+        red_ratio = max(0.0, 1.0 - (count / cand_for_model)) if cand_for_model > 0 else 0.0
+        profiles[prof_name] = ModelProfileResult(
+            profile_name=prof_name,
+            target_token_estimate=count,
+            measurement_source=src.value,
+            reduction_ratio=red_ratio,
+            context_tokens_avoided=avoided,
+        )
+
+    # 3. Router Overhead Benchmark
+    router = DeterministicModelRouter()
+    fake_prov = DeterministicFakeProvider()
+    providers = {fake_prov.provider_id: fake_prov}
+    req = ModelRequest(user_prompt=query, compiled_context=compiled)
+
+    r_start = time.perf_counter()
+    iterations = 50
+    for _ in range(iterations):
+        await router.route(req, providers, policy=RoutingPolicy.LOCAL_FIRST)
+    router_overhead = ((time.perf_counter() - r_start) * 1000.0) / iterations
+
+    # 4. Telemetry Persistence Overhead Benchmark
+    import tempfile
+    test_db_path = db_path or Path(tempfile.mkdtemp()) / "bench_telemetry.db"
+    db = Database(test_db_path)
+    await db.initialize()
+    repo = SqliteTelemetryRepository(db.connection())
+
+    t_start = time.perf_counter()
+    tel_iterations = 20
+    for i in range(tel_iterations):
+        sample_tel = ModelInvocationTelemetry(
+            invocation_id=uuid4(),
+            provider_id="fake",
+            model_id="fake-default",
+            is_local=True,
+            candidate_context_tokens=candidate_tokens,
+            compiled_context_tokens=compiled_tokens,
+            context_tokens_avoided=max(0, candidate_tokens - compiled_tokens),
+            reduction_ratio=1.0 - (compiled_tokens / candidate_tokens) if candidate_tokens > 0 else 0.0,
+            token_measurement_source=TokenMeasurementSource.PROVIDER_REPORTED,
+            routing_policy=RoutingPolicy.LOCAL_FIRST,
+            routing_reason="benchmark",
+            selected_provider="fake",
+            selected_model="fake-default",
+            finish_reason=ModelFinishReason.STOP,
+        )
+        await repo.record(sample_tel)
+    telemetry_overhead = ((time.perf_counter() - t_start) * 1000.0) / tel_iterations
+    await db.close()
+
+    return Phase9BenchmarkReport(
+        candidate_context_tokens=candidate_tokens,
+        compiled_context_tokens=compiled_tokens,
+        profiles=profiles,
+        router_overhead_ms=router_overhead,
+        telemetry_overhead_ms=telemetry_overhead,
+        token_counting_overhead_ms=tc_overhead,
+    )
+
+
+if __name__ == "__main__":
+    report = asyncio.run(run_phase9_benchmark())
+    print("\n=== CONTEXTOS PHASE 9 BENCHMARK REPORT ===")
+    print(f"Candidate Context Tokens: {report.candidate_context_tokens}")
+    print(f"Compiled Context Tokens:  {report.compiled_context_tokens}")
+    print(f"Router Overhead:          {report.router_overhead_ms:.3f} ms")
+    print(f"Telemetry Write Overhead: {report.telemetry_overhead_ms:.3f} ms")
+    print(f"Token Counting Overhead:  {report.token_counting_overhead_ms:.3f} ms")
+    print("\n--- Target Model Tokenizer Profiles (Synthetic Profiles — NOT Actual Vendor Benchmarks) ---")
+    for name, p in report.profiles.items():
+        print(
+            f"  {name:20s}: tokens={p.target_token_estimate:4d} | "
+            f"reduction={p.reduction_ratio * 100.0:.1f}% | "
+            f"avoided={p.context_tokens_avoided:4d} | source={p.measurement_source}"
+        )