contextos-memory-runtime 1.0.0rc2__py3-none-any.whl

contextos/api/routes/system.py

@@ -0,0 +1,216 @@
+"""System status and diagnostics API routes."""
+
+from __future__ import annotations
+
+import os
+import time
+import asyncio
+
+from fastapi import APIRouter
+
+from contextos import __version__
+from contextos.api.server import get_service
+from contextos.core.models import MemoryFilters, SystemStatus, TokenStats
+from contextos.core.enums import MemoryStatus
+
+router = APIRouter(tags=["system"])
+
+_start_time = time.time()
+
+
+@router.get("/status", response_model=SystemStatus)
+async def status() -> SystemStatus:
+    """Get system health and status."""
+    database = get_service("database")
+    embedding_service = get_service("embedding")
+    vector_store = get_service("vector_store")
+    bm25_index = get_service("bm25_index")
+
+    # Get counts
+    from contextos.core.protocols import MemoryRepository
+    memory_repo: MemoryRepository = get_service("memory_repo")
+    total_count = await memory_repo.count()
+    active_count = await memory_repo.count(MemoryFilters(status=MemoryStatus.ACTIVE))
+
+    from contextos.core.protocols import EventRepository
+    event_repo: EventRepository = get_service("event_repo")
+    event_count = await event_repo.count()
+
+    vector_count = await vector_store.count()
+    bm25_count = await bm25_index.count()
+
+    db_size = await database.get_size_bytes()
+
+    return SystemStatus(
+        daemon_running=True,
+        pid=os.getpid(),
+        uptime_seconds=time.time() - _start_time,
+        total_memories=total_count,
+        active_memories=active_count,
+        total_events=event_count,
+        embedding_model=embedding_service.model_name,
+        embedding_model_loaded=True,
+        vector_index_size=vector_count,
+        bm25_index_size=bm25_count,
+        database_size_bytes=db_size,
+        data_directory="[LOCAL]",
+    )
+
+
+@router.get("/stats", response_model=TokenStats)
+async def stats() -> TokenStats:
+    """Report stored counts and observed model compilations with provenance."""
+    conn = get_service("database").connection()
+    cursor = await conn.execute(
+        "SELECT COALESCE(SUM(token_count), 0), COUNT(*) FROM memories WHERE status = ?",
+        (MemoryStatus.ACTIVE.value,),
+    )
+    total_tokens, memory_count = await cursor.fetchone()
+    bases = await get_service("telemetry_query").context_measurement_bases()
+    basis = bases[0] if len(bases) == 1 and bases[0]["source"] != "unknown" else None
+    if basis is not None:
+        cursor = await conn.execute(
+            "SELECT COUNT(*), COALESCE(SUM(compiled_context_tokens), 0), "
+            "COALESCE(SUM(context_tokens_avoided), 0), "
+            "COALESCE(SUM(candidate_context_tokens), 0) "
+            "FROM model_invocations WHERE status = 'success'"
+        )
+        runs, compiled, avoided, candidates = await cursor.fetchone()
+    else:
+        runs = compiled = avoided = candidates = None
+    from contextos.api.routes.desktop import _public_label
+
+    return TokenStats(
+        total_tokens_stored=total_tokens,
+        tokens_per_memory=total_tokens / memory_count if memory_count else 0,
+        # Standalone compilations are not persisted; these are successful model runs.
+        total_compilations=runs,
+        total_tokens_compiled=compiled,
+        total_tokens_saved=avoided,
+        average_compression_ratio=avoided / candidates if candidates else None,
+        measurement_basis=f"{basis['source']}:{_public_label(basis['tokenizer'])}" if basis else None,
+    )
+
+
+@router.post("/doctor")
+async def doctor() -> dict:
+    """Run diagnostic checks."""
+    results: dict = {
+        "version": __version__,
+        "checks": {},
+    }
+
+    # Database integrity
+    database = get_service("database")
+    ok, msg = await database.integrity_check()
+    results["checks"]["database_integrity"] = {"ok": ok, "message": msg}
+
+    from contextos.storage.database import SCHEMA_VERSION
+    conn = database.connection()
+    cursor = await conn.execute("SELECT MAX(version) FROM schema_version")
+    version_row = await cursor.fetchone()
+    schema_version = version_row[0] if version_row else None
+    results["checks"]["schema"] = {
+        "ok": schema_version == SCHEMA_VERSION,
+        "version": schema_version, "expected": SCHEMA_VERSION,
+    }
+    cursor = await conn.execute(
+        "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN "
+        "('memories', 'memory_relations', 'model_invocations', 'connector_state', 'graph_nodes', 'graph_edges')"
+    )
+    tables = {row[0] for row in await cursor.fetchall()}
+    results["checks"]["tables"] = {"ok": len(tables) == 6, "present": sorted(tables)}
+    cursor = await conn.execute("PRAGMA index_list('memories')")
+    indexes = {row[1] for row in await cursor.fetchall()}
+    required_indexes = {"idx_memories_status", "idx_memories_slot_key"}
+    results["checks"]["indexes"] = {
+        "ok": required_indexes <= indexes,
+        "required_present": sorted(indexes & required_indexes),
+    }
+    graph_repo = get_service("graph_repo")
+    graph_nodes, graph_edges, _ = await graph_repo.counts()
+    graph_dirty = await graph_repo.source_is_dirty()
+    results["checks"]["graph_projection"] = {
+        "ok": True, "dirty": graph_dirty, "nodes": graph_nodes, "edges": graph_edges,
+        "suggestion": "Graph rebuilds on the next graph retrieval" if graph_dirty else None,
+    }
+    results["checks"]["telemetry"] = {
+        "ok": True, "recorded_invocations": await get_service("telemetry_repo").count(),
+    }
+    connector_ids = get_service("connectors").list_connectors()
+    connector_states = await get_service("connector_repo").list_states()
+    results["checks"]["connectors"] = {
+        "ok": True, "configured": len(connector_ids),
+        "failed": sum(state.status == "failed" for state in connector_states),
+    }
+    settings = get_service("settings")
+    results["checks"]["local_configuration"] = {
+        "ok": settings.daemon.host in {"127.0.0.1", "localhost", "::1"}
+              and os.access(database.path.parent, os.R_OK | os.W_OK),
+        "data_directory_accessible": os.access(database.path.parent, os.R_OK | os.W_OK),
+    }
+    scan = get_service("secret_scanner").scan("Authorization: Bearer privatefixture1234567890")
+    results["checks"]["privacy_scanner"] = {"ok": bool(scan.matches)}
+    providers = get_service("providers")
+
+    async def available(provider):
+        try:
+            return bool(await asyncio.wait_for(provider.health(), timeout=1.0))
+        except Exception:
+            return False
+
+    healths = await asyncio.gather(*(available(provider) for provider in providers.values()))
+    results["checks"]["providers"] = {
+        "ok": any(healths), "configured": len(providers), "available": sum(healths),
+        "optional_unavailable": len(providers) - sum(healths),
+    }
+
+    # Embedding model
+    try:
+        embedding = get_service("embedding")
+        dim = embedding.dimension
+        results["checks"]["embedding_model"] = {
+            "ok": True,
+            "model": embedding.model_name,
+            "dimension": dim,
+        }
+    except Exception as e:
+        results["checks"]["embedding_model"] = {"ok": False, "error": e.__class__.__name__}
+
+    # Index consistency
+    memory_repo = get_service("memory_repo")
+    vector_store = get_service("vector_store")
+    bm25_index = get_service("bm25_index")
+
+    from contextos.services.retrieval_index import INDEXED_STATUSES
+    indexed_count = 0
+    for memory_status in INDEXED_STATUSES:
+        indexed_count += await memory_repo.count(MemoryFilters(status=memory_status))
+    vector_count = await vector_store.count()
+    bm25_count = await bm25_index.count()
+
+    results["checks"]["index_consistency"] = {
+        "ok": True,
+        "indexable_memories": indexed_count,
+        "vector_index": vector_count,
+        "bm25_index": bm25_count,
+        "warnings": [],
+    }
+
+    if vector_count != indexed_count:
+        results["checks"]["index_consistency"]["warnings"].append(
+            f"Vector index ({vector_count}) != indexable memories ({indexed_count})"
+        )
+    if bm25_count != indexed_count:
+        results["checks"]["index_consistency"]["warnings"].append(
+            f"BM25 index ({bm25_count}) != indexable memories ({indexed_count})"
+        )
+
+    if results["checks"]["index_consistency"]["warnings"]:
+        results["checks"]["index_consistency"]["ok"] = False
+
+    results["overall"] = all(
+        c.get("ok", False) for c in results["checks"].values()
+    )
+
+    return results

contextos/api/server.py

@@ -0,0 +1,195 @@
+"""FastAPI server setup for ContextOS.
+
+This is the HTTP API that the CLI and future MCP server communicate with.
+All business logic lives in the service layer — this is a thin translation
+layer between HTTP and service protocol calls.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from contextlib import asynccontextmanager
+from typing import Any
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+
+from contextos import __version__
+from contextos.core.exceptions import (
+    ContextOSError,
+    InvalidTransitionError,
+    MemoryNotFoundError,
+    SecretDetectedError,
+)
+
+logger = logging.getLogger(__name__)
+
+# Service instances — set by daemon wiring before startup
+_services: dict[str, Any] = {}
+
+
+def set_services(services: dict[str, Any]) -> None:
+    """Set service instances. Called by daemon wiring at startup."""
+    _services.update(services)
+
+
+def get_service(name: str) -> Any:
+    """Get a service by name. Raises if not set."""
+    if name not in _services:
+        raise RuntimeError(f"Service '{name}' not initialized")
+    return _services[name]
+
+
+def _dispatch_error_payload(exc: Exception) -> dict[str, Any]:
+    """Expose bounded dispatch evidence without echoing provider request content."""
+    evidence = getattr(exc, "dispatch_evidence", None)
+    if evidence is None:
+        return {}
+    return {"dispatch_evidence": evidence.model_dump(mode="json")}
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan handler."""
+    logger.info("ContextOS API server starting (v%s)", __version__)
+    yield
+    logger.info("ContextOS API server shutting down")
+
+
+def create_app() -> FastAPI:
+    """Create and configure the FastAPI application."""
+    app = FastAPI(
+        title="ContextOS",
+        description="Local-first personal AI memory runtime",
+        version=__version__,
+        lifespan=lifespan,
+        docs_url="/api/docs",
+        redoc_url="/api/redoc",
+    )
+
+    # --- Middleware ---
+
+    @app.middleware("http")
+    async def request_timing(request: Request, call_next):
+        """Log request timing."""
+        start = time.perf_counter()
+        response = await call_next(request)
+        elapsed = (time.perf_counter() - start) * 1000
+        logger.debug(
+            "%s %s — %d (%.1fms)",
+            request.method, request.url.path, response.status_code, elapsed,
+        )
+        return response
+
+    # --- Exception Handlers ---
+
+    @app.exception_handler(RequestValidationError)
+    async def request_validation_handler(request: Request, exc: RequestValidationError):
+        """Return validation structure without echoing attacker-controlled values."""
+        safe_errors = [
+            {key: value for key, value in error.items() if key not in {"input", "ctx"}}
+            for error in exc.errors()
+        ]
+        return JSONResponse(status_code=422, content={"detail": safe_errors})
+
+    @app.exception_handler(MemoryNotFoundError)
+    async def memory_not_found_handler(request: Request, exc: MemoryNotFoundError):
+        return JSONResponse(status_code=404, content={"error": str(exc)})
+
+    @app.exception_handler(InvalidTransitionError)
+    async def invalid_transition_handler(request: Request, exc: InvalidTransitionError):
+        return JSONResponse(status_code=422, content={"error": str(exc)})
+
+    @app.exception_handler(SecretDetectedError)
+    async def secret_detected_handler(request: Request, exc: SecretDetectedError):
+        return JSONResponse(
+            status_code=422,
+            content={
+                "error": str(exc),
+                "secret_types": exc.secret_types,
+            },
+        )
+
+    from contextos.core.exceptions import (
+        ContextWindowExceededError,
+        MalformedProviderResponseError,
+        ModelUnavailableError,
+        ProviderAuthenticationError,
+        ProviderRateLimitError,
+        ProviderTimeoutError,
+        ProviderUnavailableError,
+        RoutingFailureError,
+    )
+
+    @app.exception_handler(ProviderUnavailableError)
+    async def provider_unavailable_handler(request: Request, exc: ProviderUnavailableError):
+        return JSONResponse(status_code=503, content={"error": str(exc), "provider_id": exc.provider_id, **_dispatch_error_payload(exc)})
+
+    @app.exception_handler(ModelUnavailableError)
+    async def model_unavailable_handler(request: Request, exc: ModelUnavailableError):
+        return JSONResponse(status_code=404, content={"error": str(exc), "model_id": exc.model_id, **_dispatch_error_payload(exc)})
+
+    @app.exception_handler(ContextWindowExceededError)
+    async def context_window_handler(request: Request, exc: ContextWindowExceededError):
+        return JSONResponse(
+            status_code=400,
+            content={
+                "error": str(exc),
+                "model_id": exc.model_id,
+                "required_tokens": exc.required_tokens,
+                "context_window": exc.context_window,
+                "prompt_tokens": exc.prompt_tokens,
+                "compiled_context_tokens": exc.compiled_context_tokens,
+                "reserved_output_tokens": exc.reserved_output_tokens,
+                **_dispatch_error_payload(exc),
+            },
+        )
+
+    @app.exception_handler(MalformedProviderResponseError)
+    async def malformed_provider_handler(request: Request, exc: MalformedProviderResponseError):
+        return JSONResponse(
+            status_code=502,
+            content={"error": str(exc), "provider_id": exc.provider_id, **_dispatch_error_payload(exc)},
+        )
+
+    @app.exception_handler(ProviderTimeoutError)
+    async def provider_timeout_handler(request: Request, exc: ProviderTimeoutError):
+        return JSONResponse(status_code=504, content={"error": str(exc), **_dispatch_error_payload(exc)})
+
+    @app.exception_handler(ProviderAuthenticationError)
+    async def provider_auth_handler(request: Request, exc: ProviderAuthenticationError):
+        return JSONResponse(status_code=401, content={"error": str(exc), **_dispatch_error_payload(exc)})
+
+    @app.exception_handler(ProviderRateLimitError)
+    async def provider_rate_limit_handler(request: Request, exc: ProviderRateLimitError):
+        headers = {}
+        if exc.retry_after is not None:
+            headers["Retry-After"] = str(int(exc.retry_after))
+        return JSONResponse(status_code=429, content={"error": str(exc), **_dispatch_error_payload(exc)}, headers=headers)
+
+    @app.exception_handler(RoutingFailureError)
+    async def routing_failure_handler(request: Request, exc: RoutingFailureError):
+        return JSONResponse(status_code=400, content={"error": str(exc), "policy": exc.policy})
+
+    @app.exception_handler(ContextOSError)
+    async def contextos_error_handler(request: Request, exc: ContextOSError):
+        return JSONResponse(status_code=500, content={"error": str(exc)})
+
+    # --- Register Routes ---
+    from contextos.api.routes.ingest import router as ingest_router
+    from contextos.api.routes.memories import router as memories_router
+    from contextos.api.routes.models import router as models_router
+    from contextos.api.routes.retrieval import router as retrieval_router
+    from contextos.api.routes.system import router as system_router
+    from contextos.api.routes.desktop import router as desktop_router
+
+    app.include_router(ingest_router, prefix="/api/v1")
+    app.include_router(memories_router, prefix="/api/v1")
+    app.include_router(retrieval_router, prefix="/api/v1")
+    app.include_router(models_router, prefix="/api/v1")
+    app.include_router(system_router, prefix="/api/v1")
+    app.include_router(desktop_router, prefix="/api/v1")
+
+    return app

contextos/benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Reproducible ContextOS benchmarks."""

contextos/benchmarks/compilation.py

@@ -0,0 +1,245 @@
+"""Deterministic Phase 6 compiler evaluation and scale benchmark."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from dataclasses import dataclass
+from uuid import UUID
+
+from contextos.core.enums import (
+    CompilationStrategy,
+    CompressionLevel,
+    MemoryStatus,
+    MemoryType,
+)
+from contextos.core.models import CompilationConfig, CompiledContext, Memory, ScoredMemory
+from contextos.services.compilation import QueryAwareContextCompiler, fact_is_supported
+from contextos.services.optimization import information_tokens, redundancy_similarity
+from contextos.services.token_counter import DeterministicWordTokenCounter
+
+
+SMOKE_QUERY = "Which local model should I use given my machine and previous attempts?"
+UNIT_TERMS = {
+    "runtime": {"ollama"},
+    "failed_large": {"qwen", "30b", "failed", "memory"},
+    "successful_small": {"qwen", "9b", "successfully"},
+}
+UNIT_WEIGHTS = {"runtime": 1.0, "failed_large": 1.3, "successful_small": 1.0}
+
+
+@dataclass(frozen=True)
+class CompilationMetrics:
+    input_tokens: int
+    output_tokens: int
+    compression_ratio: float
+    information_recall: float
+    weighted_preservation: float
+    unsupported_fact_rate: float
+    redundancy: float
+    budget_violations: int
+    provenance_coverage: float
+
+
+def smoke_memories() -> list[ScoredMemory]:
+    contents = [
+        "User currently uses Ollama for local model inference.",
+        (
+            "During a previous attempt the user tried Qwen 30B locally, but model "
+            "loading failed because the machine did not have enough available memory."
+        ),
+        "User successfully runs Qwen 9B locally.",
+        "User prefers concise technical responses.",
+        (
+            "Historical debugging notes discussed terminal colors and repeated setup steps. "
+            "The logs included unrelated package installation details and console output. "
+            "During a previous attempt the user tried Qwen 30B locally, but model "
+            "loading failed because the machine did not have enough available memory. "
+            "Additional historical notes repeated the same failure without new evidence. "
+            "The session ended after reviewing unrelated shell configuration."
+        ),
+    ]
+    return [
+        ScoredMemory(
+            memory=Memory(
+                id=UUID(f"40000000-0000-0000-0000-{index:012d}"),
+                content=content,
+                status=MemoryStatus.HISTORICAL if index == 5 else MemoryStatus.ACTIVE,
+                type=MemoryType.PREFERENCE if index == 4 else MemoryType.FACT,
+                confidence=0.9,
+                importance=0.8,
+            ),
+            final_score=1.0 - index * 0.05,
+            rank=index,
+            retrieval_sources=["lexical", "dense"],
+        )
+        for index, content in enumerate(contents, 1)
+    ]
+
+
+def information_unit_recall(context: CompiledContext) -> float:
+    emitted = information_tokens(" ".join(fact.text for fact in context.facts))
+    covered = sum(terms <= emitted for terms in UNIT_TERMS.values())
+    return covered / len(UNIT_TERMS)
+
+
+def weighted_preservation(context: CompiledContext) -> float:
+    emitted = information_tokens(" ".join(fact.text for fact in context.facts))
+    total = sum(UNIT_WEIGHTS.values())
+    preserved = sum(
+        weight for unit, weight in UNIT_WEIGHTS.items() if UNIT_TERMS[unit] <= emitted
+    )
+    return preserved / total
+
+
+def unsupported_fact_rate(
+    context: CompiledContext, source_memories: list[ScoredMemory]
+) -> float:
+    if not context.facts:
+        return 0.0
+    sources = {item.memory.id: item.memory.content for item in source_memories}
+    unsupported = 0
+    for fact in context.facts:
+        supported = any(
+            fact_is_supported(fact.text, sources.get(source_id, ""))
+            for source_id in fact.source_memory_ids
+        )
+        unsupported += not supported
+    return unsupported / len(context.facts)
+
+
+def fact_redundancy(context: CompiledContext) -> float:
+    if len(context.facts) < 2:
+        return 0.0
+    redundant_pairs = 0
+    pair_count = 0
+    for index, left in enumerate(context.facts):
+        for right in context.facts[index + 1:]:
+            pair_count += 1
+            similarity = redundancy_similarity(
+                information_tokens(left.text),
+                information_tokens(right.text),
+            )
+            redundant_pairs += similarity >= 0.75
+    return redundant_pairs / pair_count
+
+
+def provenance_coverage(context: CompiledContext) -> float:
+    if not context.facts:
+        return 1.0
+    return sum(
+        bool(context.provenance_map.get(fact.fact_id)) for fact in context.facts
+    ) / len(context.facts)
+
+
+def budget_violation_rate(contexts: list[CompiledContext]) -> float:
+    if not contexts:
+        return 0.0
+    return sum(context.total_tokens > context.budget for context in contexts) / len(contexts)
+
+
+def measure(
+    context: CompiledContext, source_memories: list[ScoredMemory]
+) -> CompilationMetrics:
+    return CompilationMetrics(
+        input_tokens=context.input_tokens,
+        output_tokens=context.total_tokens,
+        compression_ratio=context.compression_ratio,
+        information_recall=information_unit_recall(context),
+        weighted_preservation=weighted_preservation(context),
+        unsupported_fact_rate=unsupported_fact_rate(context, source_memories),
+        redundancy=fact_redundancy(context),
+        budget_violations=int(context.total_tokens > context.budget),
+        provenance_coverage=provenance_coverage(context),
+    )
+
+
+def synthetic_memories(count: int = 1_000) -> list[ScoredMemory]:
+    return [
+        ScoredMemory(
+            memory=Memory(
+                id=UUID(f"50000000-0000-0000-0000-{index:012d}"),
+                content=(
+                    f"Project topic{index} currently has constraint group{index % 41}. "
+                    f"Unrelated note category{index % 73} is archived."
+                ),
+                status=MemoryStatus.ACTIVE,
+                type=MemoryType.PROJECT,
+            ),
+            final_score=1.0 / (index + 1),
+            rank=index + 1,
+        )
+        for index in range(count)
+    ]
+
+
+async def main() -> None:
+    compiler = QueryAwareContextCompiler(
+        token_counter=DeterministicWordTokenCounter()
+    )
+    memories = smoke_memories()
+    strategies = (
+        CompilationStrategy.RAW_CONCAT,
+        CompilationStrategy.DEDUP_ONLY,
+        CompilationStrategy.CONTEXTOS_COMPILER,
+    )
+    print(
+        "Budget  Strategy             In  Out  Ratio  Recall  Weighted  "
+        "Unsupported  Redundancy  Violations  Provenance"
+    )
+    results: dict[tuple[int, CompilationStrategy], CompiledContext] = {}
+    for budget in (40, 60, 100, 200):
+        for strategy in strategies:
+            context = await compiler.compile(
+                SMOKE_QUERY,
+                memories,
+                CompilationConfig(
+                    budget=budget,
+                    strategy=strategy,
+                    compression_level=CompressionLevel.LIGHT,
+                ),
+            )
+            results[(budget, strategy)] = context
+            metrics = measure(context, memories)
+            print(
+                f"{budget:>6}  {strategy.value:<20} "
+                f"{metrics.input_tokens:>3}  {metrics.output_tokens:>3}  "
+                f"{metrics.compression_ratio:>5.3f}  "
+                f"{metrics.information_recall:>6.3f}  "
+                f"{metrics.weighted_preservation:>8.3f}  "
+                f"{metrics.unsupported_fact_rate:>11.3f}  "
+                f"{metrics.redundancy:>10.3f}  "
+                f"{metrics.budget_violations:>10}  "
+                f"{metrics.provenance_coverage:>10.3f}"
+            )
+
+    print("\nExact compiled contexts")
+    for budget in (40, 60, 100, 200):
+        print(f"\nBudget {budget}")
+        for strategy in strategies:
+            text = results[(budget, strategy)].context_text or "<empty>"
+            print(f"[{strategy.value}]\n{text}")
+
+    scale = synthetic_memories()
+    started = time.perf_counter()
+    first = await compiler.compile(
+        "What project constraints are current?",
+        scale,
+        CompilationConfig(budget=500),
+    )
+    latency_ms = (time.perf_counter() - started) * 1000
+    second = await compiler.compile(
+        "What project constraints are current?",
+        scale,
+        CompilationConfig(budget=500),
+    )
+    print("\nSynthetic scale")
+    print(
+        f"inputs=1000 facts={len(first.facts)} tokens={first.total_tokens}/500 "
+        f"latency_ms={latency_ms:.3f} "
+        f"deterministic={first.context_text == second.context_text}"
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())