icycache 0.3.2__py3-none-any.whl

icycache/src/upstream_client.py

@@ -0,0 +1,98 @@
+"""Client upstream résilient (async) — corrige BUG-001.
+
+Adapté du delta DELTA-002 Volet A en async pour matcher mcp_server.py (httpx.AsyncClient).
+
+Invariants :
+  1. Toute réponse (succès OU erreur) est drainée avant retour au pool.
+  2. Toute erreur de transport → connexion jetée (recycle) + 1 retry.
+  3. Après un 4xx/5xx, l'upstream est marqué "suspect" TTL court ; on sonde
+     /v1/models avant de réutiliser le pool.
+  4. Le message d'erreur porte status + body (fini le message vide).
+"""
+from __future__ import annotations
+import asyncio
+import time
+import httpx
+
+
+class UpstreamError(Exception):
+    def __init__(self, upstream: str, status: int, body: str, cause: BaseException | None = None):
+        self.upstream = upstream
+        self.status = status
+        self.body = body[:2000]
+        self.cause = cause
+        super().__init__(f"{upstream} returned {status}: {self.body[:200]!r} (cause={cause!r})")
+
+
+class ResilientUpstream:
+    """Client async thread-safe-ish (un instance par upstream, usage single-task)."""
+
+    def __init__(self, base_url: str, timeout_s: float = 60.0,
+                 connect_s: float = 10.0, suspect_ttl_s: float = 5.0):
+        self._base = base_url.rstrip("/")
+        self._timeout = httpx.Timeout(timeout_s, connect=connect_s)
+        self._client: httpx.AsyncClient | None = None
+        self._suspect_until = 0.0
+        self._suspect_ttl = suspect_ttl_s
+        self._lock = asyncio.Lock()
+
+    async def _ensure_client(self) -> httpx.AsyncClient:
+        if self._client is None or self._client.is_closed:
+            self._client = httpx.AsyncClient(timeout=self._timeout)
+        return self._client
+
+    async def _recycle(self) -> None:
+        if self._client and not self._client.is_closed:
+            try:
+                await self._client.aclose()
+            except Exception:
+                pass
+        self._client = httpx.AsyncClient(timeout=self._timeout)
+
+    async def _probe(self) -> bool:
+        try:
+            c = await self._ensure_client()
+            r = await c.get(f"{self._base}/v1/models", timeout=5.0)
+            await r.aread()
+            r.close()
+            return True
+        except Exception:
+            return False
+
+    async def _ensure_healthy(self) -> None:
+        if time.monotonic() < self._suspect_until:
+            if await self._probe():
+                self._suspect_until = 0.0
+            else:
+                await self._recycle()
+
+    def _mark_suspect(self) -> None:
+        self._suspect_until = time.monotonic() + self._suspect_ttl
+
+    async def post_json(self, path: str, json_body: dict, stream: bool = False,
+                        headers: dict | None = None) -> httpx.Response:
+        async with self._lock:
+            await self._ensure_healthy()
+            url = f"{self._base}{path}"
+            client = await self._ensure_client()
+            try:
+                resp = await client.post(url, json=json_body, headers=headers or {})
+            except (httpx.RemoteProtocolError, httpx.ReadError,
+                    httpx.ConnectError, httpx.WriteError) as e:
+                await self._recycle()
+                client = await self._ensure_client()
+                resp = await client.post(url, json=json_body, headers=headers or {})
+
+            if resp.status_code >= 400:
+                body = (await resp.aread()).decode("utf-8", "replace")
+                await resp.aclose()
+                self._mark_suspect()
+                raise UpstreamError(self._base, resp.status_code, body)
+
+            if not stream:
+                await resp.aread()
+            return resp
+
+    async def close(self) -> None:
+        if self._client and not self._client.is_closed:
+            await self._client.aclose()

icycache/src/upstream_utils.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""IcyCache upstream helpers.
+
+Extraits de prefix_proxy.py SANS effets de bord a l'import (pas de logging.basicConfig,
+pas d'init DB, pas de client httpx). Ne stocke aucune cle API.
+"""
+import os
+import json
+import hashlib
+
+
+def _flag(name, default="1"):
+    return os.environ.get(name, default) == "1"
+
+
+INJECT_ANTHROPIC = _flag("PREFIX_INJECT_ANTHROPIC")      # ajoute cache_control racine
+INJECT_OPENAI_KEY = _flag("PREFIX_INJECT_OPENAI_KEY")    # ajoute prompt_cache_key (upstream "openai" seulement)
+
+# Headers client transmis tels quels a l'upstream (aucune cle stockee cote IcyCache).
+FWD_HEADERS = {"authorization", "x-api-key", "anthropic-version", "anthropic-beta", "content-type",
+               "openai-organization", "openai-project", "http-referer", "x-title", "user-agent"}
+
+
+def usage_from(kind, u):
+    """Normalise le champ usage upstream. Valeurs absentes = None (jamais estimees)."""
+    u = u or {}
+    if kind == "anthropic":
+        i = u.get("input_tokens")
+        r = u.get("cache_read_input_tokens") or 0
+        w = u.get("cache_creation_input_tokens") or 0
+        total_in = (i or 0) + r + w if (i is not None or r or w) else None
+        return {"prompt_tokens": total_in, "completion_tokens": u.get("output_tokens"),
+                "cached_tokens": r, "cache_write_tokens": w}
+    d = u.get("prompt_tokens_details") or {}
+    return {"prompt_tokens": u.get("prompt_tokens"), "completion_tokens": u.get("completion_tokens"),
+            "cached_tokens": d.get("cached_tokens") or 0, "cache_write_tokens": d.get("cache_write_tokens") or 0}
+
+
+def _inject_anthropic_cache_control(body):
+    """Pose `cache_control` sur les blocs stables UNIQUEMENT (system + tools), jamais sur le dernier content.
+
+    Anthropic n'accepte `cache_control` que sur un bloc de contenu : un
+    `cache_control` racine fait rejeter la requete (400 extra inputs).
+    On cible UNIQUEMENT le system et les tools (prefixes stables) — jamais le
+    dernier message user/assistant, qui peut etre reecrit par la compaction et
+    invalider tout le cache Anthropic (invalidation en cascade du prefixe)."""
+    system = body.get("system")
+    if isinstance(system, str) and system.strip():
+        body["system"] = [{"type": "text", "text": system,
+                           "cache_control": {"type": "ephemeral"}}]
+    elif isinstance(system, list) and system:
+        last = system[-1]
+        if isinstance(last, dict) and "cache_control" not in last:
+            last["cache_control"] = {"type": "ephemeral"}
+    # Tools : posent aussi un breakpoint (prefixe stable, max 4 breakpoints Anthropic)
+    for t in body.get("tools") or []:
+        if isinstance(t, dict) and "cache_control" not in t:
+            t["cache_control"] = {"type": "ephemeral"}
+    return bool(system or body.get("tools"))
+
+
+def _extract_text(msg):
+    """Extrait le texte d'un message : content str ou liste de blocs (multimodal)."""
+    c = msg.get("content")
+    if isinstance(c, str):
+        return c
+    if isinstance(c, list):
+        return " ".join(b.get("text", "") for b in c if isinstance(b, dict) and "text" in b)
+    return ""
+
+
+def mutate(kind, upstream, body):
+    """Active le cache de prefixe cote fournisseur. Retourne le nom du champ injecte (ou None).
+    Anthropic : cache_control au niveau bloc (system/content). Tous les autres (famille
+    OpenAI-compatible : OpenAI, Mistral, DeepSeek, Groq, OpenRouter, omniroute, locaux) :
+    prompt_cache_key stable par session = hash(system + 1er message user + modele). Le 1er user
+    reste stable sur tous les tours d'une conversation, ce qui colle les tours au meme prefix
+    cache cote fournisseur (tokens caches factures ~10% au lieu de 100%)."""
+    if kind == "anthropic":
+        if INJECT_ANTHROPIC and "cache_control" not in json.dumps(body):
+            if _inject_anthropic_cache_control(body):
+                return "cache_control"
+    else:
+        if INJECT_OPENAI_KEY and "prompt_cache_key" not in body:
+            msgs = body.get("messages") or []
+            sys_txt = next((_extract_text(m) for m in msgs if m.get("role") in ("system", "developer")), "")
+            first_user = next((_extract_text(m) for m in msgs if m.get("role") == "user"), "")
+            if sys_txt or first_user:
+                model = body.get("model") or ""
+                src = json.dumps({"s": sys_txt, "u": first_user, "m": model},
+                                 sort_keys=True, ensure_ascii=False)
+                digest = hashlib.sha256(src.encode("utf-8")).hexdigest()[:16]
+                body["prompt_cache_key"] = "ic-" + digest
+                # session_id forward : si le client envoie un session_id (sticky routing OpenRouter/omniroute),
+                # on le laisse passer tel quel. Les providers qui ne le connaissent pas l'ignorent (sans danger).
+                return "prompt_cache_key"
+    return None
+
+
+class StreamMeter:
+    """Lit les evenements SSE au passage (sans les modifier) pour extraire usage + modele.
+
+    `events` = nombre d'evenements data recus (compteur de CHUNKS, pas de tokens).
+    """
+
+    def __init__(self, kind):
+        self.kind, self.buf, self.usage, self.model, self.events = kind, b"", {}, None, 0
+
+    def feed(self, chunk):
+        self.buf += chunk
+        while b"\n" in self.buf:
+            line, self.buf = self.buf.split(b"\n", 1)
+            line = line.strip()
+            if not line.startswith(b"data:"):
+                continue
+            raw = line[5:].strip()
+            if raw == b"[DONE]":
+                continue
+            try:
+                ev = json.loads(raw)
+            except Exception:
+                continue
+            self.events += 1
+            if self.kind == "anthropic":
+                t = ev.get("type")
+                if t == "message_start":
+                    m = ev.get("message") or {}
+                    self.model = m.get("model") or self.model
+                    self.usage.update({k: v for k, v in (m.get("usage") or {}).items() if v is not None})
+                elif t == "message_delta":
+                    self.usage.update({k: v for k, v in (ev.get("usage") or {}).items() if v is not None})
+            else:
+                self.model = ev.get("model") or self.model
+                if ev.get("usage"):
+                    self.usage = ev["usage"]