contextos-memory-runtime 1.0.0rc2__py3-none-any.whl

contextos/__init__.py

@@ -0,0 +1,3 @@
+"""ContextOS — Local-first personal AI memory runtime."""
+
+__version__ = "1.0.0rc2"

contextos/__main__.py

@@ -0,0 +1,6 @@
+"""ContextOS entry point: python -m contextos."""
+
+from contextos.cli.app import main
+
+if __name__ == "__main__":
+    main()

contextos/api/__init__.py

@@ -0,0 +1 @@
+"""API package for ContextOS."""

contextos/api/routes/__init__.py

@@ -0,0 +1 @@
+"""API routes package for ContextOS."""

contextos/api/routes/desktop.py

@@ -0,0 +1,322 @@
+"""Bounded, privacy-conscious terminal product endpoints."""
+
+from __future__ import annotations
+
+import asyncio
+import re
+import time
+from datetime import UTC, datetime, timedelta
+from typing import Any, Literal
+from uuid import UUID  # noqa: TC003 -- FastAPI resolves UUID route parameters at runtime.
+
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel, Field
+
+from contextos.api.server import get_service
+from contextos.core.enums import MemoryStatus, SourceRole
+from contextos.core.models import IngestRequest, MemoryFilters, ModelCapabilities
+
+router = APIRouter(tags=["terminal"])
+_model_cache: tuple[float, object, list[dict[str, Any]]] | None = None
+
+
+def _public_label(value: object) -> str:
+    """Keep identifiers useful while refusing credential, URI and path shapes."""
+    from contextos.services.explainability import safe_text
+    clean = safe_text(value, 160)
+    if re.search(
+        r"(?i)(?:[a-z]:[\\/]|\\\\|[a-z][a-z0-9+.-]*://|"
+        r"\b(?:api[_-]?key|password|secret|bearer|authorization|cookie|credential)\b|"
+        r"sk-[a-z0-9_-]{8,}|(?:^|\s)/(?:[^/\s]+/)+)", clean,
+    ):
+        return "[REDACTED]"
+    return clean
+
+
+def _measurement_label(source: str | None) -> str:
+    if source in {"provider_reported", "tokenizer_counted"}:
+        return "MEASURED"
+    if source == "approximated":
+        return "APPROXIMATED"
+    return "UNKNOWN"
+
+
+class RememberRequest(BaseModel):
+    text: str = Field(min_length=1, max_length=10_000, repr=False)
+
+
+@router.get("/dashboard")
+async def dashboard(
+    model: str | None = Query(default=None, max_length=200),
+    provider: str | None = Query(default=None, max_length=200),
+    period: Literal["all", "today", "week"] = "all",
+) -> dict[str, Any]:
+    repo = get_service("memory_repo")
+    telemetry = get_service("telemetry_query")
+    connectors = get_service("connectors")
+    states = {
+        state.connector_id: state
+        for state in await get_service("connector_repo").list_states()
+    }
+    tracked_cursor = await get_service("database").connection().execute(
+        "SELECT connector_id, COUNT(*) FROM connector_items WHERE deleted = 0 GROUP BY connector_id"
+    )
+    tracked_items = {row[0]: row[1] for row in await tracked_cursor.fetchall()}
+    now = datetime.now(UTC)
+    start = (
+        datetime(now.year, now.month, now.day, tzinfo=UTC) if period == "today"
+        else now - timedelta(days=7) if period == "week" else None
+    )
+    recent = await telemetry.list_recent(limit=8, model_id=model, provider_id=provider, start=start)
+    summary = await telemetry.summary_range(start=start, provider_id=provider,
+                                            model_id=model, success_only=True)
+    breakdown = await telemetry.provider_model_breakdown(start, provider, model)
+    summary_data = summary.model_dump(mode="json")
+    for group in ("by_provider", "by_model"):
+        summary_data[group] = {
+            _public_label(key): value for key, value in summary_data[group].items()
+        }
+    for row in breakdown:
+        row["provider"] = _public_label(row["provider"])
+        row["model"] = _public_label(row["model"])
+        row["context_tokenizer"] = _public_label(row["context_tokenizer"])
+    global _model_cache
+    providers = get_service("providers")
+    if (
+        _model_cache is None
+        or _model_cache[1] is not providers
+        or time.monotonic() >= _model_cache[0]
+    ):
+        async def discover(provider: Any) -> list[ModelCapabilities]:
+            try:
+                inventory = await asyncio.wait_for(provider.list_models(), timeout=1.0)
+                if not isinstance(inventory, list):
+                    return []
+                return [item for item in inventory if isinstance(item, ModelCapabilities)]
+            except Exception:
+                return []
+        inventories = await asyncio.gather(*(discover(p) for p in providers.values()))
+        discovered = [
+            {"provider": _public_label(item.provider_id), "model": _public_label(item.model_id),
+             "local": item.local, "enabled": item.enabled,
+             "simulated": item.provider_id == "fake"}
+            for inventory in inventories for item in inventory
+        ][:50]
+        _model_cache = (time.monotonic() + 10.0, providers, discovered)
+    discovered = _model_cache[2]
+    mcp = get_service("settings").mcp
+    return {
+        "memories": {
+            "active": await repo.count(MemoryFilters(status=MemoryStatus.ACTIVE)),
+            "historical": await repo.count(MemoryFilters(status=MemoryStatus.HISTORICAL)),
+            "expired": await repo.count(MemoryFilters(status=MemoryStatus.EXPIRED)),
+        },
+        "temporal": {
+            "superseded": await repo.count(MemoryFilters(status=MemoryStatus.SUPERSEDED)),
+            "contradicted": await repo.count(MemoryFilters(status=MemoryStatus.CONTRADICTED)),
+        },
+        "graph": await graph_stats(),
+        "mcp": {"enabled": mcp.enabled, "transport": mcp.transport if mcp.enabled else None,
+                "read": mcp.allow_read if mcp.enabled else False,
+                "write": mcp.allow_write if mcp.enabled else False},
+        "connectors": [
+            {"id": _public_label(cid), "status": states[cid].status if cid in states else "idle",
+             "enabled": states[cid].enabled if cid in states else True,
+             "error_code": states[cid].error_code if cid in states else None,
+             "type": _public_label(states[cid].connector_type) if cid in states else "unknown",
+             "tracked_items": tracked_items.get(cid, 0),
+             "last_success_at": states[cid].last_success_at if cid in states else None,
+             "last_attempt_at": states[cid].last_attempt_at if cid in states else None,
+             "cursor_present": bool(states[cid].cursor) if cid in states else False}
+            for cid in connectors.list_connectors()
+        ],
+        "models": [
+            item for item in discovered
+            if (model is None or item["model"] == _public_label(model))
+            and (provider is None or item["provider"] == _public_label(provider))
+        ],
+        "summary": summary_data,
+        "provider_models": breakdown,
+        "period": period,
+        "context_measurement_bases": [
+            {"source": row["source"], "tokenizer": _public_label(row["tokenizer"])}
+            for row in await telemetry.context_measurement_bases(model, provider, start)
+        ],
+        "recent": [
+            {
+             "id": str(row.invocation_id), "timestamp": row.timestamp.isoformat(),
+             "provider": _public_label(row.provider_id), "model": _public_label(row.model_id),
+             "status": row.status,
+             "preflight_input_tokens": row.preflight_input_tokens,
+             "provider_input_tokens": row.provider_input_tokens,
+             "provider_output_tokens": row.provider_output_tokens,
+             "provider_measurement_source": row.token_measurement_source.value,
+             "provider_measurement_label": _measurement_label(row.token_measurement_source.value),
+             "context_measurement_source": (row.context_token_measurement_source.value
+                                            if row.context_token_measurement_source else "unknown"),
+             "context_measurement_label": _measurement_label(
+                 row.context_token_measurement_source.value
+                 if row.context_token_measurement_source else None
+             ),
+             "context_tokenizer": _public_label(row.context_tokenizer),
+             "candidate_context_tokens": row.candidate_context_tokens,
+             "compiled_context_tokens": row.compiled_context_tokens,
+             "context_tokens_avoided": row.context_tokens_avoided,
+             "lexical_candidate_count": row.lexical_candidate_count,
+             "dense_candidate_count": row.dense_candidate_count,
+             "selected_memory_count": row.selected_memory_count,
+             "graph_expanded_count": row.graph_expanded_count,
+             "retrieval_ms": row.retrieval_ms, "compilation_ms": row.compilation_ms,
+             "provider_ms": row.provider_latency_ms, "error_code": _public_label(row.error_code)}
+            for row in recent
+        ],
+    }
+
+
+@router.get("/graph/stats")
+async def graph_stats() -> dict[str, Any]:
+    """Read projection counts without rebuilding or exposing node labels."""
+    repo = get_service("graph_repo")
+    nodes, edges, supports = await repo.counts()
+    conn = get_service("database").connection()
+    node_cursor = await conn.execute(
+        "SELECT node_type, COUNT(*) FROM graph_nodes GROUP BY node_type"
+    )
+    edge_cursor = await conn.execute(
+        "SELECT relation_type, COUNT(*) FROM graph_edges GROUP BY relation_type"
+    )
+    return {
+        "nodes": nodes, "edges": edges, "supports": supports,
+        "dirty": await repo.source_is_dirty(),
+        "average_total_degree": round(2 * edges / nodes, 3) if nodes else None,
+        "node_types": {row[0]: row[1] for row in await node_cursor.fetchall()},
+        "edge_types": {row[0]: row[1] for row in await edge_cursor.fetchall()},
+    }
+
+
+async def _graph_paths(query: str, seed_id: UUID | None = None) -> dict[str, Any]:
+    expansion = await get_service("graph").expand(
+        query_text=query, seed_memory_ids=[seed_id] if seed_id else [],
+        max_hops=3, max_nodes=50, max_edges=100,
+    )
+    candidates = sorted(expansion.candidate_scores,
+                        key=lambda item: (-expansion.candidate_scores[item], str(item)))[:10]
+    return {
+        "candidate_count": len(expansion.candidate_scores),
+        "candidates": [{
+            "memory_id": str(memory_id),
+            "graph_score": expansion.candidate_scores[memory_id],
+            "paths": [
+                path.model_dump(mode="json")
+                for path in expansion.candidate_paths.get(memory_id, [])[:5]
+            ],
+        } for memory_id in candidates],
+        "truncated": len(expansion.candidate_scores) > 10,
+    }
+
+
+@router.get("/graph/search")
+async def graph_search(entity: str = Query(min_length=1, max_length=128)) -> dict[str, Any]:
+    return await _graph_paths(entity)
+
+
+@router.get("/graph/show/{memory_id}")
+async def graph_show(memory_id: UUID) -> dict[str, Any]:
+    return await _graph_paths("", memory_id)
+
+
+@router.get("/temporal/current")
+async def temporal_current(limit: int = Query(default=25, ge=1, le=50)) -> dict[str, Any]:
+    return await _temporal_rows("active", limit)
+
+
+@router.get("/temporal/conflicts")
+async def temporal_conflicts(limit: int = Query(default=25, ge=1, le=50)) -> dict[str, Any]:
+    return await _temporal_rows("contradicted", limit)
+
+
+async def _temporal_rows(status: str, limit: int) -> dict[str, Any]:
+    conn = get_service("database").connection()
+    cursor = await conn.execute(
+        "SELECT id, status, temporal_status, observed_at, valid_from, superseded_by "
+        "FROM memories WHERE status = ? ORDER BY observed_at DESC, id DESC LIMIT ?",
+        (status, limit),
+    )
+    rows = await cursor.fetchall()
+    return {"status": status, "memories": [
+        {"memory_id": row[0], "lifecycle": row[1], "temporal_status": row[2],
+         "observed_at": row[3], "effective_at": row[4], "superseded_by": row[5]}
+        for row in rows
+    ]}
+
+
+@router.get("/temporal/history/{memory_id}")
+async def temporal_history(memory_id: UUID, include_content: bool = False) -> dict[str, Any]:
+    memory = await get_service("memory_repo").get(memory_id)
+    if memory is None:
+        raise HTTPException(status_code=404, detail="Memory not found")
+    if memory.slot is None:
+        return {"memory_id": str(memory_id), "history": [], "relations": []}
+    conn = get_service("database").connection()
+    cursor = await conn.execute(
+        "SELECT id, status, temporal_status, observed_at, valid_from, superseded_by, content "
+        "FROM memories WHERE slot_key = ? "
+        "ORDER BY COALESCE(valid_from, observed_at, created_at) DESC, id DESC LIMIT 50",
+        (memory.slot.key,),
+    )
+    from contextos.services.explainability import safe_text
+    history = []
+    for row in await cursor.fetchall():
+        item = {"memory_id": row[0], "lifecycle": row[1], "temporal_status": row[2],
+                "observed_at": row[3], "effective_at": row[4], "superseded_by": row[5]}
+        if include_content:
+            item["content"] = safe_text(row[6], 1000)
+        history.append(item)
+    evidence = await get_service("explainability").temporal_resolver.resolve_memory_evidence(memory)
+    return {"memory_id": str(memory_id), "history": history,
+            "relations": evidence["relations"][:50]}
+
+
+@router.get("/connectors")
+async def list_connectors() -> list[dict[str, Any]]:
+    manager = get_service("connectors")
+    states = {item.connector_id: item for item in await get_service("connector_repo").list_states()}
+    return [{"id": _public_label(cid), "registered": True,
+             "status": states[cid].status if cid in states else "idle",
+             "enabled": states[cid].enabled if cid in states else True,
+             "error_code": states[cid].error_code if cid in states else None,
+             "last_success_at": states[cid].last_success_at if cid in states else None,
+             "last_attempt_at": states[cid].last_attempt_at if cid in states else None}
+            for cid in manager.list_connectors()]
+
+
+@router.post("/connectors/{connector_id}/sync")
+async def sync_connector(connector_id: str) -> dict[str, Any]:
+    manager = get_service("connectors")
+    if connector_id not in manager.list_connectors():
+        raise HTTPException(status_code=404, detail="Connector is not registered")
+    result = await manager.sync(connector_id)
+    return dict(result.model_dump(mode="json"))
+
+
+@router.post("/remember")
+async def remember(request: RememberRequest) -> dict[str, Any]:
+    result = await get_service("ingestion").ingest(
+        IngestRequest(content=request.text, source_type="cli", source_role=SourceRole.USER)
+    )
+    accepted: list[str] = []
+    for candidate in result.candidates:
+        try:
+            resolved = await get_service("temporal").accept(
+                candidate, provenance_event_id=result.event_id
+            )
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            raise HTTPException(status_code=500, detail={
+                "error": "PARTIAL_WRITE" if accepted else "MEMORY_WRITE_FAILED",
+                "accepted_ids": accepted,
+            }) from None
+        accepted.append(str(resolved.memory.id))
+    return {"accepted_ids": accepted, "count": len(accepted),
+            "secrets_detected": result.secrets_detected}

contextos/api/routes/ingest.py

@@ -0,0 +1,17 @@
+"""Ingestion API routes."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from contextos.api.server import get_service
+from contextos.core.models import IngestRequest, IngestResult
+
+router = APIRouter(tags=["ingestion"])
+
+
+@router.post("/ingest", response_model=IngestResult)
+async def ingest(request: IngestRequest) -> IngestResult:
+    """Ingest content into ContextOS."""
+    ingestion_service = get_service("ingestion")
+    return await ingestion_service.ingest(request)

contextos/api/routes/memories.py

@@ -0,0 +1,84 @@
+"""Memory management API routes."""
+
+from __future__ import annotations
+
+from uuid import UUID
+
+from fastapi import APIRouter, Query
+
+from contextos.api.server import get_service
+from contextos.core.enums import MemoryStatus, MemoryType, PrivacyLevel
+from contextos.core.models import Memory, MemoryFilters, MemoryUpdate
+
+router = APIRouter(tags=["memories"])
+
+
+@router.get("/memories", response_model=list[Memory])
+async def list_memories(
+    status: MemoryStatus | None = None,
+    type: MemoryType | None = None,
+    privacy_level: PrivacyLevel | None = None,
+    source_type: str | None = None,
+    min_confidence: float | None = None,
+    min_importance: float | None = None,
+    limit: int = Query(default=50, ge=1, le=500),
+    offset: int = Query(default=0, ge=0),
+) -> list[Memory]:
+    """List memories with optional filters."""
+    memory_service = get_service("memory")
+    filters = MemoryFilters(
+        status=status,
+        type=type,
+        privacy_level=privacy_level,
+        source_type=source_type,
+        min_confidence=min_confidence,
+        min_importance=min_importance,
+        limit=limit,
+        offset=offset,
+    )
+    return await memory_service.list(filters)
+
+
+@router.get("/memories/{memory_id}", response_model=Memory)
+async def get_memory(memory_id: UUID) -> Memory:
+    """Get a specific memory by ID."""
+    memory_service = get_service("memory")
+    memory = await memory_service.get(memory_id)
+    if memory is None:
+        from contextos.core.exceptions import MemoryNotFoundError
+        raise MemoryNotFoundError(str(memory_id))
+    return memory
+
+
+@router.patch("/memories/{memory_id}", response_model=Memory)
+async def update_memory(memory_id: UUID, update: MemoryUpdate) -> Memory:
+    """Update a memory's content or metadata."""
+    memory_service = get_service("memory")
+    return await memory_service.update(memory_id, update)
+
+
+@router.post("/memories/{memory_id}/transition", response_model=Memory)
+async def transition_memory(
+    memory_id: UUID,
+    new_status: MemoryStatus,
+    reason: str = "",
+) -> Memory:
+    """Transition a memory to a new lifecycle state."""
+    memory_service = get_service("memory")
+    return await memory_service.transition(memory_id, new_status, reason)
+
+
+@router.delete("/memories/{memory_id}")
+async def delete_memory(memory_id: UUID) -> dict:
+    """Soft-delete a memory."""
+    memory_service = get_service("memory")
+    await memory_service.delete(memory_id)
+    return {"status": "deleted", "memory_id": str(memory_id)}
+
+
+@router.delete("/memories/{memory_id}/purge")
+async def purge_memory(memory_id: UUID) -> dict:
+    """Hard-delete a memory (irreversible)."""
+    memory_service = get_service("memory")
+    await memory_service.purge(memory_id)
+    return {"status": "purged", "memory_id": str(memory_id)}

contextos/api/routes/models.py

@@ -0,0 +1,81 @@
+"""API routes for model inference, capabilities, and telemetry."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, Query
+from pydantic import BaseModel, Field
+
+from contextos.api.server import get_service
+from contextos.core.enums import RoutingPolicy
+from contextos.core.models import (
+    AskResult,
+    CompilationConfig,
+    ModelCapabilities,
+    RetrievalConfig,
+    TelemetrySummary,
+)
+
+router = APIRouter(tags=["models"])
+
+
+class AskApiRequest(BaseModel):
+    """Request payload for the high-level ContextOS ask endpoint."""
+
+    query: str = Field(min_length=1)
+    system_prompt: str | None = None
+    provider: str | None = None
+    model: str | None = None
+    routing_policy: RoutingPolicy | None = None
+    allow_fallback: bool = False
+    temperature: float = Field(default=0.7, ge=0.0, le=2.0)
+    max_output_tokens: int = Field(default=1024, ge=1)
+    timeout_seconds: float = Field(default=30.0, ge=0.5, le=600.0)
+    retrieval_config: RetrievalConfig | None = None
+    compilation_config: CompilationConfig | None = None
+    session_id: str | None = None
+
+
+@router.post("/ask", response_model=AskResult)
+async def ask_model(request: AskApiRequest) -> AskResult:
+    """Prepare context, route to chosen model, execute generation, and record telemetry."""
+    model_service = get_service("model_service")
+    return await model_service.ask(
+        query=request.query,
+        system_prompt=request.system_prompt,
+        retrieval_config=request.retrieval_config,
+        compilation_config=request.compilation_config,
+        routing_policy=request.routing_policy,
+        target_provider=request.provider,
+        target_model=request.model,
+        allow_fallback=request.allow_fallback,
+        temperature=request.temperature,
+        max_output_tokens=request.max_output_tokens,
+        timeout_seconds=request.timeout_seconds,
+        session_id=request.session_id,
+    )
+
+
+@router.get("/models", response_model=list[ModelCapabilities])
+async def list_models() -> list[ModelCapabilities]:
+    """List all models offered across registered providers."""
+    providers: dict[str, Any] = get_service("providers")
+    all_models: list[ModelCapabilities] = []
+    for prov in providers.values():
+        try:
+            m_list = await prov.list_models()
+            all_models.extend(m_list)
+        except Exception:
+            continue
+    return all_models
+
+
+@router.get("/telemetry/summary", response_model=TelemetrySummary)
+async def get_telemetry_summary(
+    provider_id: str | None = Query(default=None),
+    model_id: str | None = Query(default=None),
+) -> TelemetrySummary:
+    """Get aggregated invocation metrics."""
+    telemetry_query = get_service("telemetry_query")
+    return await telemetry_query.summary_range(provider_id=provider_id, model_id=model_id)

contextos/api/routes/retrieval.py

@@ -0,0 +1,89 @@
+"""Retrieval and compilation API routes."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+from pydantic import BaseModel, Field, field_validator
+
+from contextos.api.server import get_service
+from contextos.core.models import (
+    CompiledContext,
+    CompilationConfig,
+    ContextBudget,
+    RetrievalConfig,
+    RetrievalQuery,
+    RetrievalResult,
+)
+from contextos.services.explainability import ExplanationRequest
+from contextos.services.inspection import InspectionRequest
+
+router = APIRouter(tags=["retrieval"])
+
+
+class ExplainRequest(ExplanationRequest):
+    """Bounded request for deterministic pipeline explanation."""
+
+
+@router.post("/explain")
+async def explain_context(request: ExplainRequest) -> dict:
+    return (await get_service("explainability").explain(request)).model_dump(mode="json")
+
+
+@router.post("/inspect")
+async def inspect_context(request: InspectionRequest) -> dict:
+    return (await get_service("inspector").inspect(request)).model_dump(mode="json")
+
+
+class RetrieveRequest(BaseModel):
+    """Request body for retrieval."""
+
+    query: str | RetrievalQuery
+    config: RetrievalConfig | None = None
+
+    @field_validator("query")
+    @classmethod
+    def nonblank_string_query(cls, value: str | RetrievalQuery) -> str | RetrievalQuery:
+        if isinstance(value, str) and not value.strip():
+            raise ValueError("Query cannot be blank")
+        return value
+
+
+class CompileRequest(BaseModel):
+    """Request body for compilation."""
+    query: str = Field(min_length=1)
+    config: CompilationConfig | None = None
+    retrieval_config: RetrievalConfig | None = None
+
+
+@router.post("/retrieve", response_model=RetrievalResult)
+async def retrieve(request: RetrieveRequest) -> RetrievalResult:
+    """Retrieve relevant memories for a query."""
+    retrieval_service = get_service("retrieval")
+    return await retrieval_service.retrieve(request.query, request.config)
+
+
+@router.post("/compile", response_model=CompiledContext)
+async def compile_context(request: CompileRequest) -> CompiledContext:
+    """Retrieve and compile context for a query."""
+    retrieval_service = get_service("retrieval")
+    compilation_service = get_service("compilation")
+    optimizer = get_service("optimizer")
+
+    # First retrieve
+    retrieval_result = await retrieval_service.retrieve(
+        request.query, request.retrieval_config
+    )
+
+    compilation_config = request.config or CompilationConfig()
+    selection = optimizer.optimize(
+        request.query,
+        retrieval_result.memories,
+        ContextBudget(max_tokens=compilation_config.budget),
+    )
+
+    # Carry selected memories and eligible oversized rescue candidates explicitly.
+    return await compilation_service.compile(
+        query=request.query,
+        memories=selection,
+        config=compilation_config,
+    )