devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/codex/proxy.py

@@ -0,0 +1,412 @@
+"""
+devtorch_core.codex.proxy
+=========================
+Lightweight HTTP proxy server for intercepting OpenAI Codex CLI traffic.
+
+Uses only stdlib (http.server, urllib.request, threading) — no FastAPI or
+httpx dependency required.
+
+Default port: 8766  (main DevTorch proxy runs on 8765)
+Upstream:     https://api.openai.com
+
+Routes:
+  POST /v1/*         → inject RACP, forward, capture response
+  GET  /health       → 200 {"status": "ok"}
+  *    (everything else) → transparent forward
+
+Errors always return 502 JSON — never crash the proxy.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import threading
+import urllib.error
+import urllib.request
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("devtorch.codex.proxy")
+
+DEFAULT_PORT = 8766
+DEFAULT_UPSTREAM = "https://api.openai.com"
+
+# Headers that must NOT be forwarded to the upstream (they are hop-by-hop or
+# set by the proxy itself when re-encoding the body).
+_HOP_BY_HOP = frozenset(
+    [
+        "connection",
+        "keep-alive",
+        "proxy-authenticate",
+        "proxy-authorization",
+        "te",
+        "trailers",
+        "transfer-encoding",
+        "upgrade",
+        "host",
+        "content-length",  # recalculated after body modification
+    ]
+)
+
+
+# ---------------------------------------------------------------------------
+# Request handler
+# ---------------------------------------------------------------------------
+
+class _CodexRequestHandler(BaseHTTPRequestHandler):
+    """One instance per incoming HTTP connection."""
+
+    # Injected by CodexProxy on the server object
+    upstream: str = DEFAULT_UPSTREAM
+
+    # Silence the default access log — we write our own on error
+    def log_message(self, fmt: str, *args) -> None:  # type: ignore[override]
+        pass
+
+    # ------------------------------------------------------------------
+    # Routing
+    # ------------------------------------------------------------------
+
+    def do_GET(self) -> None:
+        if self.path == "/health":
+            self._send_json(200, {"status": "ok", "proxy": "devtorch-codex"})
+            return
+        self._forward_transparent()
+
+    def do_POST(self) -> None:
+        if self.path.startswith("/v1/"):
+            self._handle_v1_post()
+        else:
+            self._forward_transparent()
+
+    # Support additional methods by transparent proxy
+    def do_PUT(self) -> None:
+        self._forward_transparent()
+
+    def do_DELETE(self) -> None:
+        self._forward_transparent()
+
+    def do_PATCH(self) -> None:
+        self._forward_transparent()
+
+    def do_HEAD(self) -> None:
+        self._forward_transparent()
+
+    def do_OPTIONS(self) -> None:
+        self._forward_transparent()
+
+    # ------------------------------------------------------------------
+    # v1 POST handler — RACP injection + capture
+    # ------------------------------------------------------------------
+
+    def _handle_v1_post(self) -> None:
+        try:
+            body_bytes = self._read_body()
+        except Exception as exc:
+            self._send_error_502(f"failed to read request body: {exc}")
+            return
+
+        # Parse JSON — if not parseable, forward raw
+        try:
+            request_body = json.loads(body_bytes) if body_bytes else {}
+        except Exception:
+            request_body = None
+
+        # Inject RACP prefix
+        modified_body_bytes = body_bytes
+        if request_body is not None:
+            try:
+                from devtorch_core.codex.capture import CodexCaptureHandler
+                import uuid
+
+                session_id = str(uuid.uuid4())
+                handler = CodexCaptureHandler()
+                modified_body = handler.pre_execute(request_body, session_id)
+                modified_body_bytes = json.dumps(modified_body).encode("utf-8")
+            except Exception as exc:
+                logger.warning("devtorch codex RACP injection failed — %s", exc)
+                session_id = ""
+                handler = None
+        else:
+            session_id = ""
+            handler = None
+
+        # Forward to upstream
+        upstream_url = self.upstream.rstrip("/") + self.path
+        fwd_headers = self._build_forward_headers(len(modified_body_bytes))
+
+        try:
+            req = urllib.request.Request(
+                upstream_url,
+                data=modified_body_bytes,
+                headers=fwd_headers,
+                method="POST",
+            )
+            with urllib.request.urlopen(req) as resp:
+                resp_bytes = resp.read()
+                resp_status = resp.status
+                resp_headers = dict(resp.headers)
+        except urllib.error.HTTPError as exc:
+            resp_bytes = exc.read()
+            resp_status = exc.code
+            resp_headers = dict(exc.headers)
+        except Exception as exc:
+            self._send_error_502(f"upstream request failed: {exc}")
+            return
+
+        # Send response back to Codex CLI
+        self._relay_response(resp_status, resp_headers, resp_bytes)
+
+        # Fire post_execute in background — never blocks the response
+        if handler is not None and request_body is not None:
+            try:
+                resp_body = json.loads(resp_bytes) if resp_bytes else {}
+            except Exception:
+                resp_body = {}
+            t = threading.Thread(
+                target=_safe_post_execute,
+                args=(handler, request_body, resp_body, session_id),
+                daemon=True,
+            )
+            t.start()
+
+    # ------------------------------------------------------------------
+    # Transparent forward
+    # ------------------------------------------------------------------
+
+    def _forward_transparent(self) -> None:
+        try:
+            body_bytes = self._read_body()
+        except Exception as exc:
+            self._send_error_502(f"failed to read request body: {exc}")
+            return
+
+        upstream_url = self.upstream.rstrip("/") + self.path
+        fwd_headers = self._build_forward_headers(len(body_bytes) if body_bytes else 0)
+
+        try:
+            req = urllib.request.Request(
+                upstream_url,
+                data=body_bytes if body_bytes else None,
+                headers=fwd_headers,
+                method=self.command,
+            )
+            with urllib.request.urlopen(req) as resp:
+                resp_bytes = resp.read()
+                resp_status = resp.status
+                resp_headers = dict(resp.headers)
+        except urllib.error.HTTPError as exc:
+            resp_bytes = exc.read()
+            resp_status = exc.code
+            resp_headers = dict(exc.headers)
+        except Exception as exc:
+            self._send_error_502(f"upstream request failed: {exc}")
+            return
+
+        self._relay_response(resp_status, resp_headers, resp_bytes)
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def _read_body(self) -> bytes:
+        length = int(self.headers.get("Content-Length", 0))
+        if length > 0:
+            return self.rfile.read(length)
+        return b""
+
+    def _build_forward_headers(self, content_length: int) -> dict:
+        """Build headers to send upstream, stripping hop-by-hop headers."""
+        headers: dict = {}
+        for key, val in self.headers.items():
+            if key.lower() not in _HOP_BY_HOP:
+                headers[key] = val
+        if content_length > 0:
+            headers["Content-Length"] = str(content_length)
+        # Ensure Host is set to the upstream host
+        from urllib.parse import urlparse
+        parsed = urlparse(self.upstream)
+        headers["Host"] = parsed.netloc
+        return headers
+
+    def _relay_response(self, status: int, headers: dict, body: bytes) -> None:
+        """Write upstream response back to the Codex CLI connection."""
+        self.send_response(status)
+        for key, val in headers.items():
+            if key.lower() not in ("transfer-encoding", "connection"):
+                self.send_header(key, val)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if body:
+            self.wfile.write(body)
+
+    def _send_json(self, status: int, payload: dict) -> None:
+        body = json.dumps(payload).encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _send_error_502(self, detail: str) -> None:
+        logger.warning("devtorch codex proxy 502 — %s", detail)
+        self._send_json(502, {"error": "bad_gateway", "detail": detail})
+
+
+# ---------------------------------------------------------------------------
+# Background post_execute helper
+# ---------------------------------------------------------------------------
+
+def _safe_post_execute(
+    handler: object,
+    request_body: dict,
+    response_body: dict,
+    session_id: str,
+) -> None:
+    try:
+        handler.post_execute(request_body, response_body, session_id)  # type: ignore[attr-defined]
+    except Exception as exc:
+        logger.warning("devtorch codex post_execute background error — %s", exc)
+
+
+# ---------------------------------------------------------------------------
+# CodexProxy
+# ---------------------------------------------------------------------------
+
+class CodexProxy:
+    """
+    Lightweight stdlib-only HTTP proxy for intercepting Codex CLI traffic.
+
+    Parameters
+    ----------
+    port:
+        Local port to bind on (default 8766).
+    upstream:
+        Base URL of the real OpenAI API (default https://api.openai.com).
+    """
+
+    def __init__(
+        self,
+        port: int = DEFAULT_PORT,
+        upstream: str = DEFAULT_UPSTREAM,
+    ) -> None:
+        self.port = port
+        self.upstream = upstream
+        self._server: Optional[HTTPServer] = None
+        self._thread: Optional[threading.Thread] = None
+
+    def start(self) -> None:
+        """Start the proxy server (blocking — press Ctrl+C to stop)."""
+        self._server = self._make_server()
+        try:
+            self._server.serve_forever()
+        except KeyboardInterrupt:
+            pass
+        finally:
+            self._server.server_close()
+
+    def start_background(self) -> threading.Thread:
+        """
+        Start the proxy in a daemon thread.
+
+        Returns the thread (already started).  Call stop() to shut down.
+        """
+        self._server = self._make_server()
+        t = threading.Thread(target=self._server.serve_forever, daemon=True)
+        t.start()
+        self._thread = t
+        return t
+
+    def stop(self) -> None:
+        """Shut down the proxy server gracefully."""
+        if self._server is not None:
+            self._server.shutdown()
+            self._server.server_close()
+            self._server = None
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _make_server(self) -> HTTPServer:
+        upstream = self.upstream
+
+        class _Handler(_CodexRequestHandler):
+            pass
+
+        _Handler.upstream = upstream  # type: ignore[assignment]
+        server = HTTPServer(("127.0.0.1", self.port), _Handler)
+        server.timeout = 30
+        return server
+
+
+# ---------------------------------------------------------------------------
+# install_codex_hook
+# ---------------------------------------------------------------------------
+
+def install_codex_hook(project_root: str) -> tuple[bool, str]:
+    """
+    Create `.codex/config.json` under *project_root* pointing at the local
+    DevTorch Codex proxy.
+
+    Idempotent — does not overwrite an existing config.
+
+    Returns (True, config_path) on success, (False, error_message) on failure.
+    """
+    try:
+        codex_dir = Path(project_root) / ".codex"
+        codex_dir.mkdir(parents=True, exist_ok=True)
+
+        config_path = codex_dir / "config.json"
+
+        # Idempotent — do not overwrite if already present
+        if config_path.exists():
+            return True, str(config_path)
+
+        config = {
+            "apiBase": f"http://localhost:{DEFAULT_PORT}/v1",
+            "provider": "openai",
+        }
+        config_path.write_text(json.dumps(config, indent=2), encoding="utf-8")
+        return True, str(config_path)
+
+    except Exception as exc:
+        return False, str(exc)
+
+
+# ---------------------------------------------------------------------------
+# get_codex_hook_status
+# ---------------------------------------------------------------------------
+
+def get_codex_hook_status(project_root: str) -> dict:
+    """
+    Return the install and runtime status of the DevTorch Codex proxy.
+
+    Returns::
+
+        {
+            "codex_config": bool,       # .codex/config.json exists
+            "codex_config_path": str,   # absolute path to config file
+            "proxy_running": bool,      # True if GET /health returns 200
+        }
+    """
+    config_path = Path(project_root) / ".codex" / "config.json"
+    codex_config = config_path.exists()
+
+    proxy_running = False
+    try:
+        req = urllib.request.Request(
+            f"http://localhost:{DEFAULT_PORT}/health",
+            method="GET",
+        )
+        with urllib.request.urlopen(req, timeout=2) as resp:
+            proxy_running = resp.status == 200
+    except Exception:
+        proxy_running = False
+
+    return {
+        "codex_config": codex_config,
+        "codex_config_path": str(config_path),
+        "proxy_running": proxy_running,
+    }

devtorch_core/concept_catalog.py

@@ -0,0 +1,209 @@
+"""
+devtorch_core.concept_catalog
+==============================
+Unified concept enumeration across all DevTorch data sources.
+
+Sources:
+    - I3 concept definitions (.GCC/concepts/*.json)
+    - Theta coordination vector (.GCC/theta.json)
+    - Sensitivity events (.GCC/sensitivities/events.jsonl)
+    - Learning trigger_concepts (.GCC/reasoning_learnings/learnings/*.json)
+    - Commit concepts (.GCC/commits/*.json)
+    - Call concepts (.GCC/reasoning_learnings/calls/*.json)
+"""
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from devtorch_core.topics import TopicStore
+
+logger = logging.getLogger("devtorch.concept_catalog")
+
+
+class ConceptCatalog:
+    """Unify all concept sources into a single enumerator."""
+
+    def __init__(self, gcc_dir: Path | str) -> None:
+        self._gcc_dir = Path(gcc_dir)
+
+    def enumerate_all(self) -> dict[str, list[str]]:
+        """Return {concept_name: [source1, source2, ...]} from all sources.
+
+        All concept names are lowercased for deduplication.
+        """
+        result: dict[str, list[str]] = {}
+
+        def _add(concept: str, source: str) -> None:
+            key = concept.lower()
+            if key not in result:
+                result[key] = []
+            if source not in result[key]:
+                result[key].append(source)
+
+        for name in self._i3_concepts():
+            _add(name, "i3_definition")
+        for name in self._theta_concepts():
+            _add(name, "theta")
+        for name in self._sensitivity_concepts():
+            _add(name, "sensitivity")
+        for name in self._learning_concepts():
+            _add(name, "learning")
+        for name in self._commit_concepts():
+            _add(name, "commit")
+        for name in self._call_concepts():
+            _add(name, "call")
+
+        return dict(sorted(result.items()))
+
+    def enumerate_learned(self) -> dict[str, dict[str, Any]]:
+        """Return {concept: {learning_count, active, stale, deprecated, success_rate, avg_confidence}}.
+
+        Only concepts that have at least one learning are included.
+        """
+        from devtorch_core.reasoning_plus.learning.analytics import concept_stats
+        from devtorch_core.reasoning_plus.learning.store import LearningStore
+
+        store = LearningStore(self._gcc_dir)
+        stats = concept_stats(store)
+        result: dict[str, dict[str, Any]] = {}
+        for cname, data in stats.items():
+            if cname == "__untyped__":
+                continue
+            result[cname] = {
+                "learning_count": data["total"],
+                "active": data["active"],
+                "stale": data["stale"],
+                "deprecated": data["deprecated"],
+                "success_rate": data["success_rate"],
+                "avg_confidence": data["avg_confidence"],
+            }
+        return result
+
+    def concepts_for_topic(
+        self, topic: str, hide_empty: bool = True,
+    ) -> dict[str, Any]:
+        """Return concepts associated with a topic.
+
+        Returns a dict with:
+            - "concepts": list of {name, sources, has_data} for concepts with data
+            - "taxonomy_only": list of concept names that have no data (if hide_empty)
+        """
+        store = TopicStore(self._gcc_dir)
+        concepts = store.get_concepts(topic)
+        if not concepts:
+            return {"concepts": [], "taxonomy_only": []}
+
+        all_concepts = self.enumerate_all()
+        learned = self.enumerate_learned()
+
+        with_data: list[dict[str, Any]] = []
+        without_data: list[str] = []
+
+        for concept in concepts:
+            sources = all_concepts.get(concept.lower(), [])
+            if sources:
+                entry: dict[str, Any] = {"name": concept, "sources": sources}
+                if concept.lower() in learned:
+                    stats = learned[concept.lower()]
+                    entry["learning_count"] = stats["learning_count"]
+                    entry["success_rate"] = stats["success_rate"]
+                    entry["avg_confidence"] = stats["avg_confidence"]
+                with_data.append(entry)
+            else:
+                without_data.append(concept)
+
+        return {
+            "concepts": with_data,
+            "taxonomy_only": without_data if hide_empty else [],
+        }
+
+    def topic_for_concept(self, concept: str) -> str | None:
+        """Reverse lookup: which topic does this concept belong to?"""
+        store = TopicStore(self._gcc_dir)
+        cname = concept.lower()
+
+        for topic in store.list_topics():
+            topic_concepts = [c.lower() for c in topic["concepts"]]
+            if cname in topic_concepts:
+                return topic["name"]
+
+        return None
+
+    def _i3_concepts(self) -> list[str]:
+        concepts_dir = self._gcc_dir / "concepts"
+        if not concepts_dir.exists():
+            return []
+        return [p.stem for p in concepts_dir.glob("*.json")]
+
+    def _theta_concepts(self) -> list[str]:
+        path = self._gcc_dir / "theta.json"
+        if not path.exists():
+            return []
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            return list(data.get("coordination_vector", {}).keys())
+        except (json.JSONDecodeError, OSError):
+            return []
+
+    def _sensitivity_concepts(self) -> list[str]:
+        path = self._gcc_dir / "sensitivities" / "events.jsonl"
+        if not path.exists():
+            return []
+        concepts: set[str] = set()
+        try:
+            for line in path.read_text(encoding="utf-8").splitlines():
+                if not line.strip():
+                    continue
+                ev = json.loads(line)
+                payload = ev.get("payload", ev)
+                concept = payload.get("target_concept", "")
+                if concept:
+                    concepts.add(concept)
+        except (json.JSONDecodeError, OSError):
+            pass
+        return list(concepts)
+
+    def _learning_concepts(self) -> list[str]:
+        path = self._gcc_dir / "reasoning_learnings" / "learnings"
+        if not path.exists():
+            return []
+        concepts: set[str] = set()
+        for p in path.glob("*.json"):
+            try:
+                data = json.loads(p.read_text(encoding="utf-8"))
+                for c in data.get("trigger_concepts", []):
+                    concepts.add(c)
+            except (json.JSONDecodeError, OSError):
+                continue
+        return list(concepts)
+
+    def _commit_concepts(self) -> list[str]:
+        path = self._gcc_dir / "commits"
+        if not path.exists():
+            return []
+        concepts: set[str] = set()
+        for p in path.glob("*.json"):
+            try:
+                data = json.loads(p.read_text(encoding="utf-8"))
+                for c in data.get("concepts", []):
+                    concepts.add(c)
+            except (json.JSONDecodeError, OSError):
+                continue
+        return list(concepts)
+
+    def _call_concepts(self) -> list[str]:
+        path = self._gcc_dir / "reasoning_learnings" / "calls"
+        if not path.exists():
+            return []
+        concepts: set[str] = set()
+        for p in path.glob("*.json"):
+            try:
+                data = json.loads(p.read_text(encoding="utf-8"))
+                for c in data.get("concepts", []):
+                    concepts.add(c)
+            except (json.JSONDecodeError, OSError):
+                continue
+        return list(concepts)

devtorch_core/consolidation/__init__.py

@@ -0,0 +1,3 @@
+from .workflow import ConsolidationRecord, ConsolidationWorkflow
+
+__all__ = ["ConsolidationRecord", "ConsolidationWorkflow"]