devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/proxy/server.py

@@ -0,0 +1,356 @@
+"""
+DevTorch local HTTP proxy (S7 + S26).
+
+Runs on localhost:8765. Routes:
+  /anthropic/*     → https://api.anthropic.com/*
+  /openai/*        → https://api.openai.com/*
+  /gemini/*        → https://generativelanguage.googleapis.com/*
+  /ollama/*        → http://localhost:11434/*
+  /groq/*          → https://api.groq.com/openai/v1/*
+  /azure-openai/*  → Azure OpenAI endpoint (from env or headers)
+
+Usage:
+  devtorch proxy start          # foreground, Ctrl+C to stop
+  devtorch proxy install        # install as OS service
+
+Traffic never routes through DevTorch servers — this is a local-only tool
+that binds to 127.0.0.1 only. Developer API keys are passed through
+unchanged to the upstream provider.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional
+
+try:
+    from fastapi import FastAPI, Request, Response
+    from fastapi.responses import StreamingResponse
+    import httpx
+    import uvicorn
+    HAS_PROXY_DEPS = True
+except ImportError:
+    HAS_PROXY_DEPS = False
+
+DEFAULT_PORT = 8765
+DEFAULT_HOST = "127.0.0.1"
+
+
+# ---------------------------------------------------------------------------
+# GCC repo discovery
+# ---------------------------------------------------------------------------
+
+def find_gcc_repo() -> Optional[Any]:
+    """
+    Walk up from cwd looking for an initialised .GCC/ directory.
+    Returns a GCCRepository instance or None if none found.
+    """
+    try:
+        from devtorch_core import GCCRepository
+        for parent in [Path.cwd()] + list(Path.cwd().parents):
+            try:
+                r = GCCRepository.at(parent)
+                if r.is_initialized():
+                    return r
+            except Exception:
+                continue
+    except Exception:
+        pass
+    return None
+
+
+# ---------------------------------------------------------------------------
+# App factory
+# ---------------------------------------------------------------------------
+
+def create_app(gcc_repo: Optional[Any] = None) -> "FastAPI":
+    """
+    Create the FastAPI proxy application.
+
+    Parameters
+    ----------
+    gcc_repo:
+        An initialised GCCRepository used for RACP injection and capture.
+        If None, requests are forwarded unmodified.
+    """
+    if not HAS_PROXY_DEPS:
+        raise ImportError(
+            "Proxy dependencies not installed. Run: pip install devtorch[proxy]"
+        )
+
+    app = FastAPI(
+        title="DevTorch Local Proxy",
+        version="0.1.0",
+        description=(
+            "Local HTTP proxy that intercepts LLM traffic, injects RACP context, "
+            "captures reasoning to .GCC/, then forwards to the real LLM API. "
+            "Binds to 127.0.0.1 only — never exposed to the network."
+        ),
+    )
+
+    # ------------------------------------------------------------------
+    # Health check
+    # ------------------------------------------------------------------
+
+    @app.get("/health")
+    async def health() -> dict:
+        initialized = False
+        try:
+            initialized = gcc_repo is not None and gcc_repo.is_initialized()
+        except Exception:
+            pass
+        return {"status": "ok", "gcc_initialized": initialized}
+
+    # ------------------------------------------------------------------
+    # Anthropic proxy
+    # ------------------------------------------------------------------
+
+    @app.api_route(
+        "/anthropic/{path:path}",
+        methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+    )
+    async def anthropic_proxy(path: str, request: Request) -> Response:
+        from .routes.anthropic import proxy_anthropic
+        return await proxy_anthropic(path, request, gcc_repo)
+
+    # ------------------------------------------------------------------
+    # OpenAI proxy
+    # ------------------------------------------------------------------
+
+    @app.api_route(
+        "/openai/{path:path}",
+        methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+    )
+    async def openai_proxy(path: str, request: Request) -> Response:
+        from .routes.openai import proxy_openai
+        return await proxy_openai(path, request, gcc_repo)
+
+    # ------------------------------------------------------------------
+    # Gemini proxy
+    # ------------------------------------------------------------------
+
+    @app.api_route(
+        "/gemini/{path:path}",
+        methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+    )
+    async def gemini_proxy(path: str, request: Request) -> Response:
+        from .routes.gemini import proxy_gemini
+        return await proxy_gemini(path, request, gcc_repo)
+
+    # ------------------------------------------------------------------
+    # Ollama proxy
+    # ------------------------------------------------------------------
+
+    @app.api_route(
+        "/ollama/{path:path}",
+        methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+    )
+    async def ollama_proxy(path: str, request: Request) -> Response:
+        from .routes.ollama import proxy_ollama
+        return await proxy_ollama(path, request, gcc_repo)
+
+    # ------------------------------------------------------------------
+    # Groq proxy
+    # ------------------------------------------------------------------
+
+    @app.api_route(
+        "/groq/{path:path}",
+        methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+    )
+    async def groq_proxy(path: str, request: Request) -> Response:
+        from .routes.groq import proxy_groq
+        return await proxy_groq(path, request, gcc_repo)
+
+    # ------------------------------------------------------------------
+    # Azure OpenAI proxy
+    # ------------------------------------------------------------------
+
+    @app.api_route(
+        "/azure-openai/{path:path}",
+        methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+    )
+    async def azure_openai_proxy(path: str, request: Request) -> Response:
+        from .routes.azure_openai import proxy_azure_openai
+        return await proxy_azure_openai(path, request, gcc_repo)
+
+    return app
+
+
+# ---------------------------------------------------------------------------
+# Proxy runner
+# ---------------------------------------------------------------------------
+
+def run_proxy(
+    port: int = DEFAULT_PORT,
+    host: str = DEFAULT_HOST,
+    gcc_repo: Optional[Any] = None,
+) -> None:
+    """
+    Start the proxy server.  Blocks until Ctrl+C.
+
+    Binds to 127.0.0.1 by default — never exposed to the network.
+    """
+    if not HAS_PROXY_DEPS:
+        raise ImportError(
+            "Proxy dependencies not installed. Run: pip install devtorch[proxy]"
+        )
+
+    repo = gcc_repo or find_gcc_repo()
+    app = create_app(repo)
+
+    print(f"DevTorch proxy listening on http://{host}:{port}")
+    if repo is not None:
+        try:
+            initialized = repo.is_initialized()
+            print(f"  .GCC/ found: {'yes' if initialized else 'no'}")
+        except Exception:
+            pass
+    else:
+        print("  No .GCC/ found — forwarding without RACP injection")
+
+    uvicorn.run(app, host=host, port=port, log_level="warning")
+
+
+# ---------------------------------------------------------------------------
+# OS service installer
+# ---------------------------------------------------------------------------
+
+def install_service() -> None:
+    """Install devtorch proxy as an OS auto-start service."""
+    import platform
+    system = platform.system()
+    if system == "Darwin":
+        _install_launchagent()
+    elif system == "Linux":
+        _install_systemd()
+    elif system == "Windows":
+        _install_windows_service()
+    else:
+        print(
+            f"Auto-install not supported on {system}. "
+            "Run `devtorch proxy start` manually."
+        )
+
+
+# ---------------------------------------------------------------------------
+# macOS LaunchAgent
+# ---------------------------------------------------------------------------
+
+def _install_launchagent() -> None:
+    """Write ~/Library/LaunchAgents/ai.devtorch.proxy.plist"""
+    import shutil
+
+    devtorch_bin = shutil.which("devtorch")
+    if devtorch_bin is None:
+        print(
+            "Could not locate devtorch executable on PATH. "
+            "Install devtorch first (`pip install devtorch[proxy]`), "
+            "then re-run `devtorch proxy install`."
+        )
+        return
+
+    plist_dir = Path.home() / "Library" / "LaunchAgents"
+    plist_dir.mkdir(parents=True, exist_ok=True)
+    plist_path = plist_dir / "ai.devtorch.proxy.plist"
+
+    plist_content = f"""\
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
+  "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>ai.devtorch.proxy</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>{devtorch_bin}</string>
+    <string>proxy</string>
+    <string>start</string>
+  </array>
+  <key>RunAtLoad</key>
+  <true/>
+  <key>KeepAlive</key>
+  <true/>
+  <key>StandardOutPath</key>
+  <string>/tmp/devtorch-proxy.log</string>
+  <key>StandardErrorPath</key>
+  <string>/tmp/devtorch-proxy.err</string>
+</dict>
+</plist>
+"""
+
+    plist_path.write_text(plist_content, encoding="utf-8")
+    print(f"Installed LaunchAgent at {plist_path}")
+    print(
+        f"Run: launchctl load {plist_path}\n"
+        "To unload: launchctl unload "
+        "~/Library/LaunchAgents/ai.devtorch.proxy.plist"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Linux systemd user service
+# ---------------------------------------------------------------------------
+
+def _install_systemd() -> None:
+    """Write ~/.config/systemd/user/devtorch-proxy.service"""
+    import shutil
+
+    devtorch_bin = shutil.which("devtorch")
+    if devtorch_bin is None:
+        print(
+            "Could not locate devtorch executable on PATH. "
+            "Install devtorch first, then re-run `devtorch proxy install`."
+        )
+        return
+
+    service_dir = Path.home() / ".config" / "systemd" / "user"
+    service_dir.mkdir(parents=True, exist_ok=True)
+    service_path = service_dir / "devtorch-proxy.service"
+
+    service_content = f"""\
+[Unit]
+Description=DevTorch Local Proxy
+After=network.target
+
+[Service]
+Type=simple
+ExecStart={devtorch_bin} proxy start
+Restart=on-failure
+RestartSec=5
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=default.target
+"""
+
+    service_path.write_text(service_content, encoding="utf-8")
+    print(f"Installed systemd user service at {service_path}")
+    print("Run: systemctl --user enable --now devtorch-proxy")
+    print("To check status: systemctl --user status devtorch-proxy")
+
+
+# ---------------------------------------------------------------------------
+# Windows Task Scheduler instructions
+# ---------------------------------------------------------------------------
+
+def _install_windows_service() -> None:
+    """Print instructions for Windows Task Scheduler."""
+    import shutil
+    devtorch_bin = shutil.which("devtorch") or "devtorch"
+
+    print("Windows auto-start setup (Task Scheduler):")
+    print("  1. Open Task Scheduler (taskschd.msc)")
+    print("  2. Create a new basic task:")
+    print(f"     Action: Start a program → {devtorch_bin} proxy start")
+    print("     Trigger: At log on")
+    print("  3. Or run from PowerShell:")
+    print(
+        f'     $action = New-ScheduledTaskAction -Execute "{devtorch_bin}" '
+        '-Argument "proxy start"'
+    )
+    print(
+        '     $trigger = New-ScheduledTaskTrigger -AtLogOn\n'
+        '     Register-ScheduledTask -TaskName "DevTorch Proxy" '
+        '-Action $action -Trigger $trigger -RunLevel Highest'
+    )

devtorch_core/query/__init__.py

@@ -0,0 +1,15 @@
+"""Hybrid grep + semantic search for `devtorch ask`."""
+
+from .grep import grep_search, GrepResult
+from .semantic import semantic_search, SemanticResult, SemanticBackend
+from .hybrid import hybrid_search, AskResult
+
+__all__ = [
+    "grep_search",
+    "GrepResult",
+    "semantic_search",
+    "SemanticResult",
+    "SemanticBackend",
+    "hybrid_search",
+    "AskResult",
+]

devtorch_core/query/grep.py

@@ -0,0 +1,181 @@
+"""Fast grep-based search over .GCC/ and project files."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+# Files inside .GCC/ that we want to search line-by-line
+_GCC_TEXT_FILES = (
+    "main.md",
+    "log.md",
+    "events.log.jsonl",
+)
+
+# Source extensions we will walk over inside the project root
+_SOURCE_SUFFIXES = (".py", ".md", ".ts", ".tsx", ".go", ".json")
+
+# Directories to skip when walking project source files
+_SKIP_DIRS = frozenset({
+    ".git", ".GCC", ".venv", "venv", "node_modules", "__pycache__",
+    ".pytest_cache", ".mypy_cache", ".tox", "build", "dist", "*.egg-info",
+})
+
+
+def _skip_dir(name: str) -> bool:
+    return name in _SKIP_DIRS or name.endswith(".egg-info")
+
+
+@dataclass(frozen=False)
+class GrepResult:
+    path: Path
+    line: int
+    snippet: str
+    score: float
+
+    def to_dict(self) -> dict:
+        return {
+            "path": str(self.path),
+            "line": self.line,
+            "snippet": self.snippet,
+            "score": self.score,
+        }
+
+
+def _compile_terms(query: str) -> list[re.Pattern]:
+    """Build case-insensitive regex patterns for each word in the query."""
+    words = [w for w in re.split(r"[^\w]+", query.lower()) if w and len(w) > 1]
+    if not words:
+        # Fallback to a non-empty pattern so we still search for single characters.
+        words = [w for w in query.lower().strip() if w.strip()] or [""]
+    return [re.compile(re.escape(word), re.IGNORECASE) for word in words]
+
+
+def _score_line(line: str, patterns: list[re.Pattern]) -> float:
+    """Return a score between 0 and 1 based on term coverage."""
+    if not patterns:
+        return 0.0
+    lower = line.lower()
+    matches = sum(1 for pat in patterns if pat.search(lower))
+    return matches / len(patterns)
+
+
+def _snip(line: str, max_len: int = 240) -> str:
+    line = line.strip()
+    if len(line) > max_len:
+        line = line[:max_len].rstrip() + "..."
+    return line
+
+
+def _collect_gcc_files(gcc_dir: Path) -> Iterable[Path]:
+    for name in _GCC_TEXT_FILES:
+        path = gcc_dir / name
+        if path.exists() and path.is_file():
+            yield path
+    # Also search concept files and sensitivity files if they exist as .md or .json
+    for subdir in ("concepts", "sensitivities"):
+        sub = gcc_dir / subdir
+        if sub.exists():
+            for path in sub.rglob("*"):
+                if path.is_file() and path.suffix in (".md", ".json", ".txt"):
+                    yield path
+
+
+def _search_file(path: Path, patterns: list[re.Pattern], min_score: float) -> list[GrepResult]:
+    results: list[GrepResult] = []
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except Exception:
+        return results
+
+    if path.name == "events.log.jsonl":
+        # Render each JSON line as a searchable text snippet
+        for line_no, raw in enumerate(text.splitlines(), start=1):
+            try:
+                payload = json.loads(raw)
+            except Exception:
+                continue
+            rendered = json.dumps(payload, ensure_ascii=False, separators=(", ", ": "))
+            score = _score_line(rendered, patterns)
+            if score >= min_score:
+                results.append(GrepResult(
+                    path=path,
+                    line=line_no,
+                    snippet=_snip(rendered),
+                    score=score,
+                ))
+    else:
+        for line_no, line in enumerate(text.splitlines(), start=1):
+            score = _score_line(line, patterns)
+            if score >= min_score:
+                results.append(GrepResult(
+                    path=path,
+                    line=line_no,
+                    snippet=_snip(line),
+                    score=score,
+                ))
+    return results
+
+
+def _collect_project_files(root: Path) -> Iterable[Path]:
+    """Yield project source files while skipping common noise directories."""
+    if not root.exists():
+        return
+
+    def _is_noise(p: Path) -> bool:
+        return p.name == ".GCC" or _skip_dir(p.name)
+
+    # Root-level files
+    for candidate in root.iterdir():
+        if candidate.is_file() and candidate.suffix in _SOURCE_SUFFIXES:
+            yield candidate
+
+    # Files inside non-noise directories
+    for path in root.iterdir():
+        if not path.is_dir() or _is_noise(path):
+            continue
+        for candidate in path.rglob("*"):
+            if candidate.is_file() and candidate.suffix in _SOURCE_SUFFIXES:
+                # Skip any nested noise dir
+                if any(_skip_dir(part) for part in candidate.parts):
+                    continue
+                yield candidate
+
+
+def grep_search(
+    query: str,
+    project_root: Path,
+    top_n: int = 5,
+    min_score: float = 0.5,
+) -> list[GrepResult]:
+    """Run a fast regex grep over .GCC/ and project source files.
+
+    Args:
+        query: free-text query
+        project_root: repository root that may contain .GCC/
+        top_n: maximum results to return
+        min_score: minimum term coverage score (0-1) for a line to be included
+    """
+    patterns = _compile_terms(query)
+    gcc_dir = project_root / ".GCC"
+    all_results: list[GrepResult] = []
+
+    if gcc_dir.exists():
+        for path in _collect_gcc_files(gcc_dir):
+            all_results.extend(_search_file(path, patterns, min_score))
+
+    for path in _collect_project_files(project_root):
+        all_results.extend(_search_file(path, patterns, min_score))
+
+    # Boost exact substring matches and stable sort by score desc
+    query_lower = query.lower()
+    for r in all_results:
+        if query_lower in r.snippet.lower():
+            # small bonus for exact substring, capped at 1.0
+            r.score = min(1.0, r.score + 0.1)
+
+    all_results.sort(key=lambda r: r.score, reverse=True)
+    return all_results[:top_n]

devtorch_core/query/hybrid.py

@@ -0,0 +1,86 @@
+"""Hybrid query combining fast grep results and optional semantic search."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from .grep import grep_search, GrepResult
+from .semantic import semantic_search, SemanticResult, SemanticBackend
+
+
+@dataclass(frozen=False)
+class AskResult:
+    path: Path
+    line: int
+    snippet: str
+    score: float
+    source: str
+
+    def to_dict(self) -> dict:
+        return {
+            "path": str(self.path),
+            "line": self.line,
+            "snippet": self.snippet,
+            "score": self.score,
+            "source": self.source,
+        }
+
+
+def hybrid_search(
+    query: str,
+    project_root: Path,
+    top_n: int = 5,
+    mode: str = "hybrid",
+    semantic_backend: SemanticBackend | None = None,
+) -> list[AskResult]:
+    """Run grep and/or semantic search, then merge and deduplicate.
+
+    Args:
+        query: free-text query
+        project_root: repository root
+        top_n: maximum number of results to return
+        mode: one of "hybrid" (default), "grep", "semantic"
+        semantic_backend: optional backend for testing / custom models
+    """
+    if mode not in {"hybrid", "grep", "semantic"}:
+        raise ValueError("mode must be one of hybrid, grep, semantic")
+
+    grep_results = []
+    semantic_results = []
+
+    if mode in ("hybrid", "grep"):
+        grep_results = grep_search(query, project_root, top_n=top_n)
+
+    if mode in ("hybrid", "semantic"):
+        semantic_results = semantic_search(query, project_root, top_n=top_n, backend=semantic_backend)
+
+    if mode == "grep":
+        return [
+            AskResult(r.path, r.line, r.snippet, r.score, "grep")
+            for r in grep_results
+        ]
+
+    if mode == "semantic":
+        return [
+            AskResult(r.path, r.line, r.snippet, r.score, "semantic")
+            for r in semantic_results
+        ]
+
+    # Hybrid mode: merge and deduplicate by (path, line) keeping the highest score
+    merged: list[AskResult] = [
+        AskResult(r.path, r.line, r.snippet, r.score, "grep")
+        for r in grep_results
+    ]
+    for r in semantic_results:
+        merged.append(AskResult(r.path, r.line, r.snippet, r.score, "semantic"))
+
+    best: dict[tuple[str, int], AskResult] = {}
+    for r in merged:
+        key = (str(r.path), r.line)
+        existing = best.get(key)
+        if existing is None or r.score > existing.score:
+            best[key] = r
+
+    final = sorted(best.values(), key=lambda r: r.score, reverse=True)
+    return final[:top_n]