coding-os 0.3.2__py3-none-any.whl

thinking_os/tracing.py

@@ -0,0 +1,235 @@
+"""Behavioral tracing (Cognition Trace Stream).
+
+Flowchart nodes emitted correspond to the node IDs in
+`docs/agent-workflow-flowchart-V1.html` so a viewer can animate exactly which
+node was hit and in which order.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import time
+import uuid
+from pathlib import Path
+from typing import Any
+
+_logger = logging.getLogger(__name__)
+_MAX_FILE_BYTES = 5 * 1024 * 1024  # 5 MB → rotate
+_TRACE_SUBDIR = "traces"
+
+# Canonical flowchart-node mapping. Keep in sync with V1.html `data-role` IDs.
+# Any `kind` emitted by the system must map to a flowchart node so replay works.
+FLOWCHART_NODES: dict[str, str] = {
+    # Entry lifecycle
+    "session_init": "n-sinit",
+    "gate_recorded": "n-gate",
+    # The Complexity Gate classification (cos_classify_prompt) — the single most
+    # user-meaningful trace event. Without this mapping it fell through to
+    # "unknown" and surfaced as a raw kind chip instead of phasing as Setup.
+    "classify": "n-gate",
+    # Cognition layer
+    "analyze_start": "n-analyzer",
+    "analyze_done": "n-analyzer",
+    "compose_done": "n-router",
+    "situation_override": "n-router",
+    "preset_matched": "n-router",
+    "composer_fallback": "n-router",
+    "hard_fallback": "n-router",
+    # Supervisor
+    "supervise_action": "n-supervisor",
+    "role_dispatch": "n-supervisor",
+    "role_output_recorded": "n-supervisor",
+    "parallel_dispatch": "n-supervisor",
+    # Dispatcher (real SDK execution, not just supervisor decision)
+    "dispatch_started": "n-supervisor",
+    "dispatch_completed": "n-supervisor",
+    # Gates
+    "ambiguity_check": "n-ambi",
+    "ambiguity_violation": "n-ambi",
+    "traceability_check": "n-trace",
+    # Loops
+    "backtrack": "n-supervisor",
+    "discovery": "n-supervisor",
+    "anti_paralysis_warn": "n-supervisor",
+    # Execution
+    "implementation": "n-impl",
+    "verification": "n-verify",
+    # Close
+    "task_done": "n-done",
+    "session_end": "n-end",
+    # Errors
+    "error": "n-supervisor",
+}
+
+
+def _trace_dir(agent_dir: Path | None = None) -> Path:
+    """Resolve trace dir generically (Rule 1 — agent-agnostic).
+
+    Resolution order: explicit arg → $COS_AGENT_DIR →
+    $COS_STATE_DIR/$COS_AGENT → fallback ".coding-os/claude".
+    """
+    if agent_dir is None:
+        import os as _os
+
+        explicit = _os.environ.get("COS_AGENT_DIR")
+        if explicit:
+            agent_dir = Path(explicit)
+        else:
+            agent = _os.environ.get("COS_AGENT") or "claude"
+            state = _os.environ.get("COS_STATE_DIR")
+            base = Path(state) if state else Path(".coding-os")
+            agent_dir = base / agent
+    d = agent_dir / _TRACE_SUBDIR
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def _trace_path(session_id: str, agent_dir: Path | None = None) -> Path:
+    return _trace_dir(agent_dir) / f"{session_id}.jsonl"
+
+
+def _rotate_if_large(path: Path) -> None:
+    """If trace file exceeds limit, rename to <session>.<ts>.jsonl and start fresh."""
+    try:
+        if path.exists() and path.stat().st_size >= _MAX_FILE_BYTES:
+            ts = time.strftime("%Y%m%dT%H%M%S", time.gmtime())
+            path.rename(path.with_suffix(f".{ts}.jsonl"))
+    except OSError as exc:
+        _logger.debug("trace rotate skipped: %s", exc)
+
+
+def emit(
+    session_id: str,
+    kind: str,
+    data: dict[str, Any] | None = None,
+    *,
+    trace_id: str | None = None,
+    span_id: str | None = None,
+    parent_span: str | None = None,
+    agent_dir: Path | None = None,
+    role: str | None = None,
+    phase: str | None = None,
+) -> dict[str, Any]:
+    try:
+        event = {
+            "ts": time.time(),
+            "session_id": session_id,
+            "trace_id": trace_id or session_id,
+            "span_id": span_id or uuid.uuid4().hex[:12],
+            "parent_span": parent_span,
+            "kind": kind,
+            "node": FLOWCHART_NODES.get(kind, "unknown"),
+            "role": role,
+            "phase": phase,
+            "data": data or {},
+        }
+        path = _trace_path(session_id, agent_dir)
+        _rotate_if_large(path)
+        line = json.dumps(event, separators=(",", ":"), default=str) + "\n"
+        _append_locked(path, line)
+        return event
+    except Exception as exc:  # pragma: no cover — tracing must never break callers
+        os.write(2, f"[tracing] failed kind={kind} exc={exc}\n".encode())
+        return {"kind": kind, "error": str(exc)}
+
+
+def _append_locked(path: Path, line: str) -> None:
+    """Flock-safe append. Each write gets its own lock for multi-agent safety."""
+    try:
+        import fcntl
+
+        with open(path, "a", encoding="utf-8") as fh:
+            try:
+                fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
+                fh.write(line)
+            finally:
+                try:
+                    fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
+                except OSError as exc:
+                    _logger.debug("trace flock release: %s", exc)
+    except ImportError:
+        # Windows — fall back to best-effort append
+        with open(path, "a", encoding="utf-8") as fh:
+            fh.write(line)
+
+
+def read_trace(session_id: str, agent_dir: Path | None = None) -> list[dict[str, Any]]:
+    """Return all events for a session in chronological order (as written)."""
+    path = _trace_path(session_id, agent_dir)
+    if not path.exists():
+        return []
+    events: list[dict[str, Any]] = []
+    try:
+        for raw in path.read_text(encoding="utf-8").splitlines():
+            raw = raw.strip()
+            if not raw:
+                continue
+            try:
+                events.append(json.loads(raw))
+            except json.JSONDecodeError:
+                events.append({"kind": "corrupt", "raw": raw})
+    except OSError:
+        return []
+    return events
+
+
+def summarize(session_id: str, agent_dir: Path | None = None) -> dict[str, Any]:
+    """
+    Return a compact summary of the trace: node-sequence, counts, roles visited,
+    presets/situations matched, violations. Useful for the `cos trace show` CLI
+    and for behavioral tests that assert "chain visited the right nodes".
+    """
+    events = read_trace(session_id, agent_dir)
+    if not events:
+        return {"session_id": session_id, "events": 0, "nodes": [], "roles": []}
+
+    nodes_sequence: list[str] = []
+    roles_visited: list[str] = []
+    kinds_counts: dict[str, int] = {}
+    presets: list[str] = []
+    situations: list[str] = []
+    chain: list[str] | None = None
+    violations: list[str] = []
+    backtracks: int = 0
+    discoveries: int = 0
+
+    for ev in events:
+        k = ev.get("kind")
+        kinds_counts[k] = kinds_counts.get(k, 0) + 1
+        node = ev.get("node")
+        if node and (not nodes_sequence or nodes_sequence[-1] != node):
+            nodes_sequence.append(node)
+        if ev.get("role"):
+            roles_visited.append(ev["role"])
+
+        data = ev.get("data") or {}
+        if k == "preset_matched":
+            presets.append(data.get("preset_id", "?"))
+        elif k == "situation_override":
+            situations.append(data.get("situation_id", "?"))
+        elif k == "compose_done":
+            chain = data.get("chain") or chain
+        elif k == "backtrack":
+            backtracks += 1
+        elif k == "discovery":
+            discoveries += 1
+        elif k == "ambiguity_violation":
+            violations.append(f"{data.get('formula', '?')}/{data.get('criterion', '?')}")
+
+    return {
+        "session_id": session_id,
+        "events": len(events),
+        "first_ts": events[0].get("ts"),
+        "last_ts": events[-1].get("ts"),
+        "nodes": nodes_sequence,
+        "roles": roles_visited,
+        "kinds": kinds_counts,
+        "preset": presets[-1] if presets else None,
+        "situation": situations[-1] if situations else None,
+        "chain": chain,
+        "violations": violations,
+        "backtracks": backtracks,
+        "discoveries": discoveries,
+    }

web/__init__.py

@@ -0,0 +1,5 @@
+"""core.web — Unified HTTP backbone for graph_os (S4)."""
+
+from .server import create_app, run_server
+
+__all__ = ["create_app", "run_server"]

web/_cache.py

@@ -0,0 +1,118 @@
+"""In-process response cache for read-only graph endpoints.
+
+Pattern: LRU + signature-based invalidation. Signature is the max
+`updated_at` over `graph_nodes` — any re-index bumps it, so cache keys
+automatically rotate when the underlying graph changes. No manual
+invalidation is required.
+
+WHY (not Redis / not memcached):
+    The Hub is a single-process uvicorn. An in-process dict is enough
+    for the working set (hot Graph-tab queries, depth-slider toggles).
+    External cache would add a deploy dep + auth surface for zero
+    gain at our scale.
+
+WHY (signature, not pure TTL):
+    A pure 60s TTL serves stale results for 60s after every reindex.
+    Signature mode goes from cached → fresh on the very next request,
+    while still hitting the cache for repeats inside that window.
+
+WHY (bounded LRU):
+    Memory ceiling. ``max_entries=256`` keeps the cache <10 MB even
+    when every entry is a 30 KB graph-export payload.
+
+THREAD SAFETY:
+    All ops behind a single ``threading.Lock``. uvicorn's worker model
+    is async + thread-pool for blocking calls, so concurrent reads
+    must be serialised through the lock — measured cost <1 µs per
+    op vs the 200 ms cache miss.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from collections import OrderedDict
+from collections.abc import Callable
+from typing import Any
+
+logger = logging.getLogger("coding_os.web.cache")
+
+
+class SignatureLRUCache:
+    """LRU cache keyed by (signature, frozen_kwargs).
+
+    ``signature_fn`` runs once per request — it must be cheap (a single
+    SQLite ``SELECT MAX(updated_at)`` is well under a millisecond on
+    the indexed column). If the signature can't be computed (DB down
+    etc.), the cache transparently falls through to the producer so
+    callers never see stale data.
+    """
+
+    def __init__(self, *, max_entries: int = 256, ttl_seconds: float = 60.0) -> None:
+        self._store: OrderedDict[tuple, tuple[float, Any]] = OrderedDict()
+        self._max = max_entries
+        self._ttl = ttl_seconds
+        self._lock = threading.Lock()
+        self._hits = 0
+        self._misses = 0
+
+    def get_or_compute(
+        self,
+        *,
+        signature_fn: Callable[[], Any],
+        cache_key: tuple,
+        producer: Callable[[], Any],
+    ) -> Any:
+        # Best-effort signature — falls through on any error so we never
+        # serve stale data when we can't tell whether it's stale.
+        try:
+            sig = signature_fn()
+        except Exception as exc:
+            logger.debug("cache signature unavailable, bypassing: %s", exc)
+            return producer()
+
+        full_key = (sig,) + tuple(cache_key)
+        now = time.monotonic()
+
+        with self._lock:
+            entry = self._store.get(full_key)
+            if entry is not None:
+                ts, value = entry
+                if (now - ts) <= self._ttl:
+                    self._store.move_to_end(full_key)
+                    self._hits += 1
+                    return value
+                # Stale — drop and recompute outside the lock.
+                del self._store[full_key]
+
+        # Produce outside the lock so concurrent misses for different
+        # keys don't serialise on each other. A double-fetch on the
+        # same key under contention is acceptable — the producer is
+        # idempotent.
+        value = producer()
+
+        with self._lock:
+            self._store[full_key] = (now, value)
+            self._store.move_to_end(full_key)
+            if len(self._store) > self._max:
+                self._store.popitem(last=False)
+            self._misses += 1
+        return value
+
+    def stats(self) -> dict[str, int]:
+        with self._lock:
+            return {
+                "size": len(self._store),
+                "hits": self._hits,
+                "misses": self._misses,
+                "max_entries": self._max,
+            }
+
+    def clear(self) -> None:
+        with self._lock:
+            self._store.clear()
+
+
+# Module-level singleton used by route handlers.
+graph_export_cache = SignatureLRUCache(max_entries=256, ttl_seconds=60.0)

web/_deps.py

@@ -0,0 +1,56 @@
+"""core.web._deps — FastAPI dependency factories."""
+
+from __future__ import annotations
+
+import sys
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from fastapi import HTTPException, Request
+
+# Ensure core/ is on sys.path when this module is imported directly.
+_CORE_DIR = Path(__file__).resolve().parent.parent
+if str(_CORE_DIR) not in sys.path:
+    sys.path.insert(0, str(_CORE_DIR))
+
+
+def _get_enterprise():
+    """Lazy import to avoid circular deps at package load time."""
+    from graph_os.enterprise import metrics, rate_limiter  # type: ignore
+
+    return rate_limiter(), metrics()
+
+
+def make_rate_limit_dep(route_name: str) -> Callable:
+    """Factory that returns a FastAPI dependency for a specific named route."""
+
+    async def _dep(request: Request) -> None:
+        limiter, _ = _get_enterprise()
+        client_ip = request.client.host if request.client else "unknown"
+        key = f"{route_name}:{client_ip}"
+        if not limiter.acquire(key):
+            raise HTTPException(
+                status_code=429,
+                detail={"error": "rate_limited", "route": route_name},
+                headers={"Retry-After": "2"},
+            )
+
+    return _dep
+
+
+def make_metrics_dep(route_name: str) -> Callable:
+    """Factory that returns a FastAPI dependency for recording per-route metrics."""
+
+    async def _dep(request: Request) -> None:
+        _, prom = _get_enterprise()
+        prom.inc_counter(f"cos_web_requests_total{{route={route_name!r}}}")
+        start = time.monotonic()
+        yield
+        duration = time.monotonic() - start
+        prom.record_timing(
+            f"cos_web_request_duration_seconds{{route={route_name!r}}}",
+            duration,
+        )
+
+    return _dep

web/_envelope.py

@@ -0,0 +1,85 @@
+"""core.web._envelope — MCP envelope → HTTP response adapter."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+
+# ----- OpenAPI schemas -------------------------------------------------
+# Documenting the standard error envelope so generated clients (e.g.
+# `openapi-typescript`) emit typed error shapes instead of `unknown`.
+class ErrorBody(BaseModel):
+    category: str = Field(
+        ...,
+        description="Error class: validation | permission | not_found | transient | unavailable | internal.",
+    )
+    message: str = Field(..., description="Human-readable explanation.")
+    retryable: bool = Field(
+        False, description="True when the caller may safely retry after a short delay."
+    )
+
+
+class ErrorEnvelope(BaseModel):
+    """Standard error envelope returned for all 4xx/5xx responses."""
+
+    error: ErrorBody
+
+
+# Reusable `responses=` block — apply to any route that returns the
+# envelope error shape (i.e. anything that goes through unwrap()).
+ENVELOPE_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = {
+    400: {"model": ErrorEnvelope, "description": "Validation error."},
+    401: {"model": ErrorEnvelope, "description": "Unauthorized (COS_HUB_TOKEN required)."},
+    403: {"model": ErrorEnvelope, "description": "Forbidden (permission / disabled module)."},
+    404: {"model": ErrorEnvelope, "description": "Resource not found."},
+    500: {"model": ErrorEnvelope, "description": "Internal error."},
+    503: {"model": ErrorEnvelope, "description": "Backend unavailable (retryable)."},
+}
+
+
+_CATEGORY_TO_STATUS: dict[str, int] = {
+    "validation": 400,
+    "unauthorized": 401,
+    "permission": 403,
+    "module_disabled": 403,  # capability gated off for this project (subsystem toggle)
+    "not_found": 404,
+    "transient": 503,
+    "unavailable": 503,
+    "internal": 500,
+}
+
+
+def unwrap(raw: str | dict[str, Any]) -> JSONResponse:
+    """Parse an MCP envelope and return the appropriate HTTP response."""
+    if isinstance(raw, str):
+        try:
+            envelope = json.loads(raw)
+        except json.JSONDecodeError:
+            return JSONResponse(
+                status_code=500,
+                content={"error": {"category": "internal", "message": "invalid tool response"}},
+            )
+    else:
+        envelope = raw
+
+    if envelope.get("ok"):
+        payload = envelope.get("data", {})
+        if isinstance(payload, dict):
+            meta = payload.pop("meta", None)
+            return JSONResponse(
+                status_code=200,
+                content={"data": payload, "meta": meta},
+            )
+        return JSONResponse(status_code=200, content={"data": payload, "meta": None})
+
+    error = envelope.get("error", {})
+    category = error.get("category", "internal")
+    status_code = _CATEGORY_TO_STATUS.get(category, 500)
+    return JSONResponse(
+        status_code=status_code,
+        content={"error": error},
+    )

web/_project_context.py

@@ -0,0 +1,140 @@
+"""core.web._project_context — per-request project scope."""
+
+from __future__ import annotations
+
+import os
+import sys
+from contextvars import ContextVar
+from pathlib import Path
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+
+# Ensure the cli/ module is importable (registry lives there, outside
+# core/web/, so the web server can call it without re-implementing the
+# schema).
+_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+
+_current_project: ContextVar[Path | None] = ContextVar(
+    "cos_current_project",
+    default=None,
+)
+
+
+def current_project_root() -> Path:
+    """Return the active project root for this request (or env fallback)."""
+    bound = _current_project.get()
+    if bound is not None:
+        return bound
+    from thinking_os.database import project_root
+
+    return project_root()
+
+
+def is_explicit_project_scope() -> bool:
+    """True when the request was routed via /api/p/<slug>/... middleware.
+
+    Routes use this to decide whether ambient COS_STATE_DIR / COS_AGENT_DIR
+    env vars are allowed to override the project's own .coding-os path.
+    Per-project URLs MUST NOT leak into a globally-set state dir.
+    """
+    return _current_project.get() is not None
+
+
+def current_db_path() -> Path:
+    """Return the active sqlite DB path (project-scoped or env fallback).
+
+    Delegates filename + env resolution to the canonical helper in
+    ``core.thinking_os.db.resolve_db_path`` — single source of truth so
+    a future filename change is one edit, not 30. Legacy `thinking_os.db`
+    is auto-renamed by `migrate_legacy_db_filename()` on first init_db().
+    """
+    from thinking_os.database import resolve_db_path  # type: ignore
+
+    bound = _current_project.get()
+    if bound is not None:
+        return resolve_db_path(bound)
+    return resolve_db_path(current_project_root())
+
+
+class ProjectScopeMiddleware(BaseHTTPMiddleware):
+    """Rewrite /api/p/<slug>/* → /api/* and bind slug's project root."""
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        path: str = request.url.path
+        if path.startswith("/api/p/"):
+            remainder = path[len("/api/p/") :]
+            slug, _, rest = remainder.partition("/")
+            if not slug:
+                return JSONResponse(
+                    {"detail": "missing project slug"},
+                    status_code=400,
+                )
+            project_root = _resolve_slug(slug)
+            if project_root is None:
+                return JSONResponse(
+                    {"detail": f"project slug {slug!r} not in registry; run `cos registry add`"},
+                    status_code=404,
+                )
+            token = _current_project.set(project_root)
+            # Mirror the scope into thinking_os so resolve_db_path() and
+            # any downstream graph_os backend lookup picks up the right
+            # project's coding-os.db. Without this, the process-wide
+            # backend singletons (graph_os.tools.graph._BACKEND_SINGLETON,
+            # graph_os.backend._BACKEND_CACHE) keep serving whichever
+            # project opened first.
+            from thinking_os.database import reset_active_project_root, set_active_project_root
+
+            db_token = set_active_project_root(project_root)
+            new_path = "/api/" + rest if rest else "/api/"
+            request.scope["path"] = new_path
+            request.scope["raw_path"] = new_path.encode("utf-8")
+            try:
+                return await call_next(request)
+            finally:
+                _current_project.reset(token)
+                reset_active_project_root(db_token)
+        return await call_next(request)
+
+
+def _resolve_slug(slug: str) -> Path | None:
+    """Look up a slug → absolute path via the global registry."""
+    try:
+        from cli.registry import load_registry  # type: ignore
+
+        reg = load_registry()
+    except Exception:
+        reg = None
+    if reg is not None:
+        for entry in reg.projects:
+            if entry.slug == slug:
+                candidate = Path(entry.path).resolve()
+                if (candidate / ".coding-os").is_dir():
+                    return candidate
+                return None
+
+    cwd = Path(os.environ.get("COS_PROJECT_ROOT") or os.getcwd()).resolve()
+    if not (cwd / ".coding-os").is_dir():
+        return None
+    # Match the same slug rule hub.py / registry use, so the auto-listed
+    # cwd entry and the middleware agree on spelling (e.g. "coding os"
+    # becomes "coding-os" in both places — not just one).
+    cwd_slug = cwd.name.lower().strip() or "project"
+    try:
+        from cli.registry import _derive_slug  # type: ignore
+
+        cwd_slug = _derive_slug(cwd)
+    except Exception as exc:
+        import logging as _logging
+
+        _logging.getLogger("coding_os.web.context").debug(
+            "cli.registry._derive_slug unavailable: %s",
+            exc,
+        )
+    if cwd_slug == slug.lower().strip():
+        return cwd
+    return None