superlocalmemory 3.6.8 → 3.6.10

package/src/superlocalmemory/optimize/proxy/_helpers.py

@@ -358,6 +358,7 @@ async def _stream_and_cache_forward(
     body_bytes: bytes,
     upstream_url: str,
     on_complete: "Callable[[bytes], Any] | None" = None,
+    max_accumulate: int | None = None,
 ) -> Response | StreamingResponse:
     """Stream-forward with optional post-stream cache-store callback.
 
@@ -388,10 +389,12 @@ async def _stream_and_cache_forward(
         )
 
     acc: list[bytes] = []
+    acc_bytes = 0
+    acc_capped = False
     complete_called = False
 
     async def _generate() -> AsyncIterator[bytes]:
-        nonlocal complete_called
+        nonlocal complete_called, acc_bytes, acc_capped
         stream_error = False
         try:
             async with proxy.http_client.stream(
@@ -399,7 +402,17 @@ async def _stream_and_cache_forward(
             ) as upstream_resp:
                 async for chunk in upstream_resp.aiter_bytes():
                     if chunk:
-                        acc.append(chunk)
+                        # Always forward to the client; only bound what we hold
+                        # in memory for the on_complete callback (CWE-400).
+                        if max_accumulate is None or acc_bytes < max_accumulate:
+                            acc.append(chunk)
+                            acc_bytes += len(chunk)
+                        elif not acc_capped:
+                            acc_capped = True
+                            logger.debug(
+                                "[%s] stream accumulator capped at %d bytes",
+                                request_id, max_accumulate,
+                            )
                         yield chunk
         except httpx.RemoteProtocolError as exc:
             stream_error = True
@@ -447,6 +460,91 @@ async def _stream_and_cache_forward(
     )
 
 
+async def capture_passthrough_forward(
+    proxy: Any,
+    request: Request,
+    *,
+    provider: str,
+    upstream_url: str,
+    allowed_headers: frozenset,
+    request_id: str,
+    model_hint: str = "",
+    sse_parser: "Callable[[bytes], bytes | None] | None" = None,
+    is_stream: bool = False,
+) -> Response | StreamingResponse:
+    """Shadow-capture passthrough (v3.6.10, plan §7).
+
+    Pure passthrough to upstream + record the exchange to the capture corpus.
+    NO cache, NO compression — capture mode observes only authentic traffic.
+    Fail-open: a capture or forward error degrades to a normal forward/error
+    response; the user's request is never blocked by capture.
+    """
+    from superlocalmemory.optimize.proxy.capture import (
+        extract_usage,
+        record_exchange_async,
+    )
+
+    body_bytes = await request.body()
+    fwd_headers = _build_forward_headers(request, allowed_headers)
+    fwd_headers["content-length"] = str(len(body_bytes))
+
+    if is_stream:
+        async def _on_complete(acc: bytes) -> None:
+            parsed = sse_parser(acc) if sse_parser else None
+            payload = parsed if parsed is not None else acc
+            itok, otok, mdl = extract_usage(provider, parsed)
+            await record_exchange_async(
+                provider=provider,
+                model=mdl or model_hint,
+                request_body=body_bytes,
+                response_body=payload,
+                content_type="text/event-stream",
+                input_tokens=itok,
+                output_tokens=otok,
+                status_code=200,
+                stream=True,
+            )
+
+        # Bound the in-memory accumulator (CWE-400): the corpus only keeps the
+        # first 1 MB per side anyway, so cap accumulation there.
+        from superlocalmemory.optimize.proxy.capture import _MAX_CAPTURE_BODY_BYTES
+        return await _stream_and_cache_forward(
+            proxy, request_id, fwd_headers, body_bytes, upstream_url,
+            on_complete=_on_complete,
+            max_accumulate=_MAX_CAPTURE_BODY_BYTES,
+        )
+
+    if proxy.http_client is None:
+        return await _fail_open_forward(proxy, request, upstream_url)
+    try:
+        upstream_resp = await proxy.http_client.post(
+            upstream_url, content=body_bytes, headers=fwd_headers,
+        )
+    except Exception as exc:
+        logger.error("[%s] capture passthrough upstream error: %r", request_id, exc)
+        return await _fail_open_forward(proxy, request, upstream_url)
+
+    resp_bytes = upstream_resp.content
+    itok, otok, mdl = extract_usage(provider, resp_bytes)
+    await record_exchange_async(
+        provider=provider,
+        model=mdl or model_hint,
+        request_body=body_bytes,
+        response_body=resp_bytes,
+        content_type="application/json",
+        input_tokens=itok,
+        output_tokens=otok,
+        status_code=upstream_resp.status_code,
+        stream=False,
+    )
+    return Response(
+        content=resp_bytes,
+        status_code=upstream_resp.status_code,
+        media_type="application/json",
+        headers=_filter_response_headers(dict(upstream_resp.headers)),
+    )
+
+
 async def _safe_cache_check(hooks: HookChain, ctx: ProxyRequest) -> CachedResponse:
     try:
         result = hooks.cache.check(ctx)

package/src/superlocalmemory/optimize/proxy/anthropic_surface.py

@@ -25,7 +25,9 @@ from superlocalmemory.optimize.proxy._helpers import (
     _safe_compress,
     _stream_and_cache_forward,
     _stream_forward,
+    capture_passthrough_forward,
 )
+from superlocalmemory.optimize.proxy.capture import capture_enabled
 from superlocalmemory.optimize.proxy.lifecycle import ProviderResponse, ProxyRequest
 
 logger = logging.getLogger("slm.optimize.proxy.anthropic")
@@ -192,6 +194,16 @@ async def handle_messages(proxy: object, request: Request) -> Response:
 
         stream = bool(body.get("stream", False))
         has_tools = _body_has_tools(body)
+
+        # v3.6.10 shadow-capture (plan §7): pure passthrough + corpus record.
+        if capture_enabled():
+            return await capture_passthrough_forward(
+                proxy, request, provider="anthropic", upstream_url=upstream_url,
+                allowed_headers=_ANTHROPIC_FORWARD_HEADERS, request_id=request_id,
+                model_hint=str(body.get("model", "")),
+                sse_parser=_parse_sse_to_json, is_stream=stream,
+            )
+
         ctx = ProxyRequest(
             provider="anthropic",
             method="POST",

package/src/superlocalmemory/optimize/proxy/capture.py

@@ -0,0 +1,243 @@
+"""capture.py — Lossless shadow-capture of real proxy traffic (v3.6.10, plan §7).
+
+Purpose: build a dogfood corpus of real {request, response, model, tokens,
+content_type} pairs so the cache + compression benchmark (benchmarks/optimize/)
+can be replayed against authentic traffic instead of only synthetic prompts.
+
+Activation: set ``SLM_OPTIMIZE_CAPTURE=1`` in the daemon's environment. When on:
+  * the proxy runs in PURE PASSTHROUGH — cache + compression hooks are disabled
+    at load time (see server._load_hooks), so capture never observes a mutated
+    request or a cache hit; every line is a genuine upstream exchange.
+  * each completed exchange is appended as one JSON line to
+    ``~/.superlocalmemory/optimize_capture.jsonl`` (0600, gitignored).
+
+ISOLATION GUARANTEE: this module writes ONLY to optimize_capture.jsonl. It never
+opens memory.db, llmcache.db, or any SLM memory store. (Plan §9 hard rule.)
+
+FAIL-OPEN: a capture failure (disk full, permission, encode error) is logged and
+swallowed — it MUST NOT break the proxied request the user is waiting on.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("slm.optimize.proxy.capture")
+
+_CAPTURE_DIRNAME = ".superlocalmemory"
+_CAPTURE_FILENAME = "optimize_capture.jsonl"
+_CAPTURE_ENV = "SLM_OPTIMIZE_CAPTURE"
+_TRUTHY = frozenset({"1", "true", "yes", "on"})
+
+# Cap a single captured body so a pathological 10 MB request can't bloat the
+# corpus line beyond what the replay harness will read back. Bodies above this
+# are recorded truncated with a marker (capture is for benchmarking, not audit).
+_MAX_CAPTURE_BODY_BYTES = 1 * 1024 * 1024  # 1 MB per side
+
+# Providers whose response bodies follow the OpenAI usage schema
+# ({"usage": {"prompt_tokens", "completion_tokens"}, "model"}).
+_OPENAI_FORMAT_PROVIDERS = frozenset({"openai", "gemini-openai-compat"})
+
+
+def capture_enabled() -> bool:
+    """True iff ``SLM_OPTIMIZE_CAPTURE`` is set to a truthy value."""
+    return os.environ.get(_CAPTURE_ENV, "").strip().lower() in _TRUTHY
+
+
+def _capture_path() -> Path:
+    return Path.home() / _CAPTURE_DIRNAME / _CAPTURE_FILENAME
+
+
+class ShadowCapture:
+    """Thread-safe append-only JSONL writer for proxy exchanges (singleton)."""
+
+    _instance: "ShadowCapture | None" = None
+    _instance_lock = threading.Lock()
+
+    def __init__(self, path: Path | None = None) -> None:
+        self._path = path or _capture_path()
+        self._write_lock = threading.Lock()
+        self._count = 0
+
+    @classmethod
+    def get_instance(cls) -> "ShadowCapture":
+        # Double-checked locking: cheap fast-path after first construction.
+        if cls._instance is None:
+            with cls._instance_lock:
+                if cls._instance is None:
+                    cls._instance = cls()
+        return cls._instance
+
+    @classmethod
+    def reset_instance(cls) -> None:
+        """Test hook — drop the singleton so a fresh path can be injected."""
+        with cls._instance_lock:
+            cls._instance = None
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    @property
+    def count(self) -> int:
+        return self._count
+
+    def record(self, entry: dict[str, Any]) -> bool:
+        """Append one capture entry as a JSON line. Returns True on success.
+
+        Fail-open: any error is logged and False is returned; never raised.
+
+        Security: opens with a single ``os.open`` carrying ``O_CREAT |
+        O_APPEND | O_NOFOLLOW`` and mode ``0o600`` on EVERY write. O_NOFOLLOW
+        refuses a symlink pre-placed at the path (symlink-append attack), and
+        the unconditional 0600-on-create removes the stat/exists TOCTOU that
+        could otherwise drop the file to the process umask.
+        """
+        try:
+            line = json.dumps(entry, ensure_ascii=False, separators=(",", ":"))
+        except (TypeError, ValueError) as exc:
+            logger.warning("capture: entry not JSON-serialisable, dropped: %r", exc)
+            return False
+
+        try:
+            with self._write_lock:
+                self._path.parent.mkdir(parents=True, exist_ok=True)
+                flags = os.O_CREAT | os.O_WRONLY | os.O_APPEND | getattr(os, "O_NOFOLLOW", 0)
+                fd = os.open(self._path, flags, 0o600)
+                with os.fdopen(fd, "a", encoding="utf-8") as fh:
+                    fh.write(line + "\n")
+                self._count += 1
+            return True
+        except OSError as exc:
+            # PermissionError / symlink-refusal (ELOOP) are security-relevant —
+            # surface the errno but still fail open so the request is never blocked.
+            logger.warning("capture: write failed (fail-open): %r", exc)
+            return False
+
+
+def _truncate(raw: bytes) -> tuple[str, bool]:
+    """Decode bytes for storage; truncate beyond the per-side cap."""
+    truncated = len(raw) > _MAX_CAPTURE_BODY_BYTES
+    head = raw[:_MAX_CAPTURE_BODY_BYTES] if truncated else raw
+    return head.decode("utf-8", errors="replace"), truncated
+
+
+def build_entry(
+    *,
+    provider: str,
+    model: str,
+    request_body: bytes,
+    response_body: bytes,
+    content_type: str,
+    input_tokens: int,
+    output_tokens: int,
+    status_code: int,
+    stream: bool,
+) -> dict[str, Any]:
+    """Construct a capture entry dict from a completed exchange.
+
+    No timestamp is stamped here (Date.now is intentionally avoided in some
+    runtimes); the replay harness keys on content, not time, and the file's own
+    line order preserves arrival sequence.
+    """
+    req_str, req_trunc = _truncate(request_body)
+    resp_str, resp_trunc = _truncate(response_body)
+    return {
+        "provider": provider,
+        "model": model,
+        "content_type": content_type,
+        "stream": stream,
+        "status_code": status_code,
+        "input_tokens": int(input_tokens),
+        "output_tokens": int(output_tokens),
+        "request": req_str,
+        "response": resp_str,
+        "request_truncated": req_trunc,
+        "response_truncated": resp_trunc,
+    }
+
+
+def extract_usage(provider: str, body: bytes | None) -> tuple[int, int, str]:
+    """Best-effort (input_tokens, output_tokens, model) from a provider JSON body.
+
+    Works on the normalised JSON the SSE parsers emit AND on non-streaming
+    upstream JSON. Returns (0, 0, "") when the body is missing/unparseable —
+    capture must never fail because usage couldn't be read.
+    """
+    if not body:
+        return 0, 0, ""
+    try:
+        data = json.loads(body)
+    except (json.JSONDecodeError, ValueError, TypeError):
+        return 0, 0, ""
+    if not isinstance(data, dict):
+        return 0, 0, ""
+
+    if provider == "gemini":
+        usage = data.get("usageMetadata") or {}
+        return (
+            int(usage.get("promptTokenCount", 0) or 0),
+            int(usage.get("candidatesTokenCount", 0) or 0),
+            str(data.get("modelVersion", "") or ""),
+        )
+
+    usage = data.get("usage") or {}
+    if provider == "anthropic":
+        return (
+            int(usage.get("input_tokens", 0) or 0),
+            int(usage.get("output_tokens", 0) or 0),
+            str(data.get("model", "") or ""),
+        )
+    # OpenAI-format providers (explicit allowlist so a future provider variant
+    # is not silently parsed with the wrong schema — it warns + returns zeros).
+    if provider not in _OPENAI_FORMAT_PROVIDERS:
+        logger.warning(
+            "capture.extract_usage: unknown provider %r — recording zero tokens", provider
+        )
+        return 0, 0, str(data.get("model", "") or "")
+    return (
+        int(usage.get("prompt_tokens", 0) or 0),
+        int(usage.get("completion_tokens", 0) or 0),
+        str(data.get("model", "") or ""),
+    )
+
+
+def record_exchange(
+    *,
+    provider: str,
+    model: str,
+    request_body: bytes,
+    response_body: bytes,
+    content_type: str = "application/json",
+    input_tokens: int = 0,
+    output_tokens: int = 0,
+    status_code: int = 200,
+    stream: bool = False,
+) -> bool:
+    """Build + append a capture entry. Fail-open. Returns True on success."""
+    entry = build_entry(
+        provider=provider,
+        model=model,
+        request_body=request_body,
+        response_body=response_body,
+        content_type=content_type,
+        input_tokens=input_tokens,
+        output_tokens=output_tokens,
+        status_code=status_code,
+        stream=stream,
+    )
+    return ShadowCapture.get_instance().record(entry)
+
+
+async def record_exchange_async(**kwargs: Any) -> bool:
+    """Async wrapper for ``record_exchange`` that offloads the synchronous file
+    write to a worker thread so it never blocks the proxy event loop — relevant
+    when many streaming responses complete in the same loop iteration.
+    """
+    return await asyncio.to_thread(lambda: record_exchange(**kwargs))

package/src/superlocalmemory/optimize/proxy/gemini_surface.py

@@ -47,7 +47,10 @@ from superlocalmemory.optimize.proxy._helpers import (
     _safe_compress,
     _stream_and_cache_forward,
     _stream_forward,
+    capture_passthrough_forward,
 )
+from superlocalmemory.optimize.proxy.capture import capture_enabled
+from superlocalmemory.optimize.proxy.openai_surface import _parse_openai_sse_to_json
 from superlocalmemory.optimize.proxy.lifecycle import ProviderResponse, ProxyRequest
 
 logger = logging.getLogger("slm.optimize.proxy.gemini")
@@ -238,6 +241,24 @@ async def handle_gemini_native(
         stream = "streamGenerateContent" in model_and_method
         has_tools = _body_has_tools(body)
 
+        # v3.6.10 shadow-capture (plan §7): pure passthrough + corpus record.
+        if capture_enabled():
+            cap_model = model_and_method.split(":", 1)[0].replace("models/", "")
+            cap_url = upstream_url
+            if stream:
+                _allowed = {
+                    k: v for k, v in request.query_params.items()
+                    if k.lower() in _GEMINI_ALLOWED_QUERY_PARAMS
+                }
+                _allowed["alt"] = "sse"
+                cap_url = f"{upstream_url}?{urllib.parse.urlencode(_allowed)}"
+            return await capture_passthrough_forward(
+                proxy, request, provider="gemini", upstream_url=cap_url,
+                allowed_headers=_GEMINI_NATIVE_FORWARD_HEADERS, request_id=request_id,
+                model_hint=cap_model,
+                sse_parser=_parse_gemini_sse_to_json, is_stream=stream,
+            )
+
         ctx = ProxyRequest(
             provider="gemini",
             method="POST",
@@ -407,6 +428,16 @@ async def handle_gemini_openai_compat(proxy: object, request: Request) -> Respon
         has_tools = _body_has_tools(body)
         stream = bool(body.get("stream", False))
 
+        # v3.6.10 shadow-capture (plan §7): pure passthrough + corpus record.
+        if capture_enabled():
+            return await capture_passthrough_forward(
+                proxy, request, provider="gemini-openai-compat",
+                upstream_url=upstream_url,
+                allowed_headers=_GEMINI_OPENAI_COMPAT_FORWARD_HEADERS,
+                request_id=request_id, model_hint=str(body.get("model", "")),
+                sse_parser=_parse_openai_sse_to_json, is_stream=stream,
+            )
+
         ctx = ProxyRequest(
             provider="gemini-openai-compat",
             method="POST",

package/src/superlocalmemory/optimize/proxy/openai_surface.py

@@ -33,7 +33,9 @@ from superlocalmemory.optimize.proxy._helpers import (
     _safe_compress,
     _stream_and_cache_forward,
     _stream_forward,
+    capture_passthrough_forward,
 )
+from superlocalmemory.optimize.proxy.capture import capture_enabled
 from superlocalmemory.optimize.proxy.lifecycle import ProviderResponse, ProxyRequest
 
 logger = logging.getLogger("slm.optimize.proxy.openai")
@@ -316,6 +318,16 @@ async def handle_chat_completions(proxy: object, request: Request) -> Response:
 
         stream = bool(body.get("stream", False))
         has_tools = _body_has_tools(body)
+
+        # v3.6.10 shadow-capture (plan §7): pure passthrough + corpus record.
+        if capture_enabled():
+            return await capture_passthrough_forward(
+                proxy, request, provider="openai", upstream_url=upstream_url,
+                allowed_headers=_OPENAI_FORWARD_HEADERS, request_id=request_id,
+                model_hint=str(body.get("model", "")),
+                sse_parser=_parse_openai_sse_to_json, is_stream=stream,
+            )
+
         ctx = ProxyRequest(
             provider="openai", method="POST", path="/v1/chat/completions",
             headers=_redact_headers(dict(request.headers)),

package/src/superlocalmemory/optimize/proxy/server.py

@@ -75,6 +75,24 @@ class ProxyApp:
             self._request_counter += 1
             return f"slm_{int(time.monotonic() * 1000)}_{self._request_counter:06d}"
 
+    def reload_from_config(self, config: OptimizeConfig) -> None:
+        """Hot-swap cache/compress behavior when optimize.json changes (v3.6.10).
+
+        Called by the ConfigStore change-callback (UI save → immediate; external
+        file/CLI edit → within the 2s watchdog poll). Rebuilds the HookChain so
+        ``cache_enabled`` and ``compress_enabled`` can be toggled INDEPENDENTLY
+        at runtime with no daemon restart. Note: ``proxy_enabled`` (whether the
+        proxy claims /v1/* at all) is a startup decision and is NOT changed here.
+        """
+        self.config = config
+        self.hooks = _load_hooks(config)
+        logger.info(
+            "slm.optimize.proxy reloaded (config v%s): cache=%s compress=%s",
+            getattr(config, "config_version", "?"),
+            type(self.hooks.cache).__name__ if self.hooks.cache else "None",
+            type(self.hooks.compress).__name__ if self.hooks.compress else "None",
+        )
+
 
 def build_proxy_router(proxy: ProxyApp) -> APIRouter:
     """Build and return the FastAPI router for all proxy surfaces."""
@@ -132,6 +150,17 @@ def build_proxy_router(proxy: ProxyApp) -> APIRouter:
 
 
 def _load_hooks(config: OptimizeConfig) -> HookChain:
+    # v3.6.10 shadow-capture (plan §7): capture mode is PURE passthrough — no
+    # cache, no compression — so the corpus records only authentic upstream
+    # exchanges. This is defense-in-depth alongside the per-surface guard.
+    from superlocalmemory.optimize.proxy.capture import capture_enabled
+    if capture_enabled():
+        logger.info(
+            "slm.optimize.proxy: SLM_OPTIMIZE_CAPTURE on — cache/compress "
+            "DISABLED, recording exchanges to optimize_capture.jsonl"
+        )
+        return HookChain.empty()
+
     cache_hook = None
     compress_hook = None