icycache 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- icycache/__init__.py +32 -0
- icycache/__main__.py +9 -0
- icycache/cli.py +171 -0
- icycache/config/icycache.gateway.json +111 -0
- icycache/config/icycache.json +149 -0
- icycache/config/normalization_rules.json +5832 -0
- icycache/config/semantic_cache.json +8 -0
- icycache/config/tools_registry.json +10 -0
- icycache/src/icy_cache_store.py +217 -0
- icycache/src/icy_core.py +380 -0
- icycache/src/icy_normalize.py +197 -0
- icycache/src/icy_paths.py +76 -0
- icycache/src/icy_semantic.py +197 -0
- icycache/src/mcp_gateway.py +360 -0
- icycache/src/mcp_server.py +1319 -0
- icycache/src/mcp_stdio_bridge.py +83 -0
- icycache/src/mcp_stdio_entry.py +23 -0
- icycache/src/prefix_proxy.py +336 -0
- icycache/src/server.py +1142 -0
- icycache/src/tool_proxy.py +655 -0
- icycache/src/upstream_client.py +98 -0
- icycache/src/upstream_utils.py +135 -0
- icycache/static/dashboard.html +2385 -0
- icycache/static/icywebgl.js +2190 -0
- icycache/static/logo.svg +58 -0
- icycache/static/three.min.js +6 -0
- icycache-0.3.2.dist-info/METADATA +422 -0
- icycache-0.3.2.dist-info/RECORD +31 -0
- icycache-0.3.2.dist-info/WHEEL +4 -0
- icycache-0.3.2.dist-info/entry_points.txt +2 -0
- icycache-0.3.2.dist-info/licenses/LICENSE +21 -0
icycache/src/server.py
ADDED
|
@@ -0,0 +1,1142 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
IcyCache - Gateway universel multi-upstream.
|
|
4
|
+
|
|
5
|
+
Point d'entree unique : http://127.0.0.1:8080
|
|
6
|
+
Tout client dont base_url pointe sur IcyCache passe par ici (Hermes, Claude Code,
|
|
7
|
+
scripts, IDE, SDK...). L'upstream de sortie est choisi EXPLICITEMENT :
|
|
8
|
+
prefixe d'URL, header X-IcyCache-Upstream, champ "upstream" du body, ou defaut de config.
|
|
9
|
+
Aucun defaut implicite vers OmniRoute : OmniRoute est UN upstream parmi d'autres.
|
|
10
|
+
|
|
11
|
+
Endpoints LLM (caches, tokens 100% upstream) :
|
|
12
|
+
POST /v1/chat/completions OpenAI-compatible
|
|
13
|
+
POST /v1/messages Anthropic natif
|
|
14
|
+
POST /v1/embeddings embeddings
|
|
15
|
+
GET /v1/models
|
|
16
|
+
ANY /{upstream}/v1/{rest} routage explicite par prefixe
|
|
17
|
+
|
|
18
|
+
Observabilite :
|
|
19
|
+
GET /ws/events WebSocket : deltas live + evenement final (tokens reels)
|
|
20
|
+
GET /health /api/status /api/metrics /api/metrics/series /api/metrics/export
|
|
21
|
+
GET /api/dashboard-stats /dashboard
|
|
22
|
+
|
|
23
|
+
Ce module NE touche PAS a static/dashboard.html ni a src/mcp_server.py : la gateway
|
|
24
|
+
tourne a cote, sur des ports isoles, jusqu'a unification explicite.
|
|
25
|
+
"""
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import asyncio
|
|
29
|
+
import hashlib
|
|
30
|
+
import json
|
|
31
|
+
import os
|
|
32
|
+
import re
|
|
33
|
+
import socket
|
|
34
|
+
import sys
|
|
35
|
+
import time
|
|
36
|
+
import uuid
|
|
37
|
+
from contextlib import asynccontextmanager
|
|
38
|
+
from datetime import datetime, timezone
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
from typing import Any, Dict, Optional, Tuple
|
|
41
|
+
|
|
42
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
43
|
+
|
|
44
|
+
from icy_paths import PROJECT_ROOT, resolve_data_path # noqa: E402
|
|
45
|
+
|
|
46
|
+
# La config de la gateway doit etre lue par icy_core aussi (meme fichier).
|
|
47
|
+
_CONFIG_FILE = Path(os.environ.get("ICYCACHE_CONFIG") or (PROJECT_ROOT / "config" / "icycache.gateway.json"))
|
|
48
|
+
os.environ["ICYCACHE_CONFIG"] = str(_CONFIG_FILE)
|
|
49
|
+
|
|
50
|
+
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect # noqa: E402
|
|
51
|
+
from fastapi.responses import ( # noqa: E402
|
|
52
|
+
FileResponse,
|
|
53
|
+
HTMLResponse,
|
|
54
|
+
JSONResponse,
|
|
55
|
+
Response,
|
|
56
|
+
StreamingResponse,
|
|
57
|
+
)
|
|
58
|
+
import httpx # noqa: E402
|
|
59
|
+
import uvicorn # noqa: E402
|
|
60
|
+
|
|
61
|
+
import icy_core # noqa: E402
|
|
62
|
+
from icy_core import hub, log_event, utc_iso # noqa: E402
|
|
63
|
+
import upstream_utils # noqa: E402
|
|
64
|
+
from upstream_utils import StreamMeter, mutate, usage_from # noqa: E402
|
|
65
|
+
from icy_cache_store import CacheStore, decode_entry, encode_entry # noqa: E402
|
|
66
|
+
import icy_normalize # noqa: E402
|
|
67
|
+
from icy_normalize import cache_key, normalize_payload # noqa: E402
|
|
68
|
+
import logging # noqa: E402
|
|
69
|
+
|
|
70
|
+
_log = logging.getLogger("IcyCache.gateway")
|
|
71
|
+
|
|
72
|
+
CONFIG: Dict[str, Any] = json.loads(_CONFIG_FILE.read_text(encoding="utf-8"))
|
|
73
|
+
DB_FILE = resolve_data_path(
|
|
74
|
+
os.environ.get("ICYCACHE_DB") or CONFIG.get("cache", {}).get("db_path"),
|
|
75
|
+
"db/cache.db",
|
|
76
|
+
)
|
|
77
|
+
DB_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
78
|
+
|
|
79
|
+
UPSTREAMS: Dict[str, Dict[str, Any]] = CONFIG.get("upstreams", {})
|
|
80
|
+
GW = CONFIG.get("gateway", {})
|
|
81
|
+
DEFAULT_UPSTREAM: Optional[str] = GW.get("default_upstream")
|
|
82
|
+
PROXY_CFG = CONFIG.get("proxy", {})
|
|
83
|
+
TIMEOUT = float(PROXY_CFG.get("timeout", 120))
|
|
84
|
+
CACHE_CFG = CONFIG.get("cache", {})
|
|
85
|
+
CACHE_UNSET_TEMPERATURE = bool(CACHE_CFG.get("cache_when_temperature_unset", True))
|
|
86
|
+
WS_DELTA_INTERVAL = float(GW.get("ws_delta_interval_s", 0.25))
|
|
87
|
+
|
|
88
|
+
CACHE = CacheStore(
|
|
89
|
+
DB_FILE,
|
|
90
|
+
ttl_seconds=int(CACHE_CFG.get("ttl_seconds", 86400)),
|
|
91
|
+
max_entries=int(CACHE_CFG.get("max_entries", 10000)),
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
CLIENT_HEADER = "x-icycache-client"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
# =============================================================================
|
|
98
|
+
# ROUTAGE
|
|
99
|
+
# =============================================================================
|
|
100
|
+
def _upstream_names() -> list:
|
|
101
|
+
return sorted(UPSTREAMS.keys())
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def resolve_upstream(request: Request, body: Optional[Dict[str, Any]] = None,
|
|
105
|
+
prefix: Optional[str] = None) -> Tuple[str, str]:
|
|
106
|
+
"""(nom_upstream, source) - jamais de defaut implicite si default_upstream absent."""
|
|
107
|
+
if prefix:
|
|
108
|
+
return _validate(prefix, "path")
|
|
109
|
+
hdr = request.headers.get("x-icycache-upstream")
|
|
110
|
+
if hdr:
|
|
111
|
+
return _validate(hdr.strip().lower(), "header")
|
|
112
|
+
if isinstance(body, dict):
|
|
113
|
+
val = body.get("upstream")
|
|
114
|
+
if isinstance(val, str) and val.strip():
|
|
115
|
+
return _validate(val.strip().lower(), "body")
|
|
116
|
+
if DEFAULT_UPSTREAM:
|
|
117
|
+
return _validate(DEFAULT_UPSTREAM, "default")
|
|
118
|
+
raise HTTPException(
|
|
119
|
+
status_code=400,
|
|
120
|
+
detail={
|
|
121
|
+
"error": "upstream_required",
|
|
122
|
+
"message": ("Aucun upstream indique. Precisez l'upstream par prefixe d'URL "
|
|
123
|
+
"(/omniroute/v1/chat/completions), par le header X-IcyCache-Upstream, "
|
|
124
|
+
"par le champ \"upstream\" du body, ou par gateway.default_upstream dans la config."),
|
|
125
|
+
"available_upstreams": _upstream_names(),
|
|
126
|
+
},
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _validate(name: str, source: str) -> Tuple[str, str]:
|
|
131
|
+
key = (name or "").strip().lower()
|
|
132
|
+
if key not in UPSTREAMS:
|
|
133
|
+
raise HTTPException(
|
|
134
|
+
status_code=404,
|
|
135
|
+
detail={"error": "unknown_upstream", "upstream": key, "source": source,
|
|
136
|
+
"available_upstreams": _upstream_names()},
|
|
137
|
+
)
|
|
138
|
+
return key, source
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def target_url(upstream: str, path: str) -> str:
|
|
142
|
+
"""Assemble base_url + chemin sans dupliquer /v1."""
|
|
143
|
+
base = str(UPSTREAMS[upstream].get("base_url", "")).rstrip("/")
|
|
144
|
+
if not base:
|
|
145
|
+
raise HTTPException(status_code=500, detail={"error": "upstream_misconfigured", "upstream": upstream})
|
|
146
|
+
if (base.endswith("/v1") or UPSTREAMS[upstream].get("strip_v1")) and path.startswith("/v1"):
|
|
147
|
+
path = path[3:] or "/"
|
|
148
|
+
return base + path
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def upstream_kind(upstream: str) -> str:
|
|
152
|
+
return str(UPSTREAMS[upstream].get("kind", "openai")).lower()
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def forward_headers(request: Request, upstream: str, kind: str) -> Dict[str, str]:
|
|
156
|
+
"""Passe les credentials du client tels quels. Aucune cle API stockee cote IcyCache."""
|
|
157
|
+
headers: Dict[str, str] = {"Content-Type": "application/json"}
|
|
158
|
+
for name in upstream_utils.FWD_HEADERS:
|
|
159
|
+
val = request.headers.get(name)
|
|
160
|
+
if val:
|
|
161
|
+
headers[name] = val
|
|
162
|
+
if "authorization" not in {k.lower() for k in headers} and "x-api-key" not in {k.lower() for k in headers}:
|
|
163
|
+
env_name = UPSTREAMS[upstream].get("api_key_env")
|
|
164
|
+
if env_name:
|
|
165
|
+
token = os.environ.get(env_name)
|
|
166
|
+
if token:
|
|
167
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
168
|
+
if kind == "anthropic" and "anthropic-version" not in {k.lower() for k in headers}:
|
|
169
|
+
headers["anthropic-version"] = request.headers.get("anthropic-version", "2023-06-01")
|
|
170
|
+
for k, v in (UPSTREAMS[upstream].get("headers") or {}).items():
|
|
171
|
+
headers[k] = v
|
|
172
|
+
headers.pop("host", None)
|
|
173
|
+
return headers
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def client_label(request: Request) -> str:
|
|
177
|
+
lab = request.headers.get(CLIENT_HEADER)
|
|
178
|
+
if lab:
|
|
179
|
+
return lab[:64]
|
|
180
|
+
ua = request.headers.get("user-agent")
|
|
181
|
+
if ua:
|
|
182
|
+
return ua[:64]
|
|
183
|
+
return "unknown"
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
# =============================================================================
|
|
187
|
+
# CACHE : CLE SUR PAYLOAD NORMALISE
|
|
188
|
+
# =============================================================================
|
|
189
|
+
def cacheable_reason(body: Dict[str, Any]) -> Tuple[bool, str]:
|
|
190
|
+
"""(cacheable, raison). Refus explicite plutot qu'un silence."""
|
|
191
|
+
if not isinstance(body, dict):
|
|
192
|
+
return False, "body invalide"
|
|
193
|
+
temp = body.get("temperature", None)
|
|
194
|
+
if temp is None:
|
|
195
|
+
if CACHE_UNSET_TEMPERATURE:
|
|
196
|
+
return True, "temperature absent (cacheable par config)"
|
|
197
|
+
return False, "temperature absent (non cacheable par config)"
|
|
198
|
+
try:
|
|
199
|
+
if float(temp) == 0.0:
|
|
200
|
+
return True, "temperature=0"
|
|
201
|
+
except (TypeError, ValueError):
|
|
202
|
+
return False, "temperature illisible"
|
|
203
|
+
return False, f"temperature={temp} (non deterministe)"
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def build_cache_key(body: Dict[str, Any], upstream: str, path: str) -> Tuple[str, str]:
|
|
207
|
+
scope = {"upstream": upstream, "path": path}
|
|
208
|
+
key = cache_key(body, scope)
|
|
209
|
+
raw = hashlib.sha256(icy_normalize.canonical(
|
|
210
|
+
{k: v for k, v in (body or {}).items() if k not in icy_normalize.CACHE_KEY_IGNORED}
|
|
211
|
+
).encode("utf-8")).hexdigest()
|
|
212
|
+
return key, raw
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def display_prompt(body: Dict[str, Any]) -> str:
|
|
216
|
+
msgs = body.get("messages") or []
|
|
217
|
+
for m in reversed(msgs):
|
|
218
|
+
if isinstance(m, dict) and m.get("role") == "user":
|
|
219
|
+
c = m.get("content")
|
|
220
|
+
if isinstance(c, str):
|
|
221
|
+
return c
|
|
222
|
+
if isinstance(c, list):
|
|
223
|
+
return " ".join(b.get("text", "") for b in c if isinstance(b, dict) and b.get("type") == "text")
|
|
224
|
+
if isinstance(body.get("system"), str):
|
|
225
|
+
return body["system"]
|
|
226
|
+
return ""
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
# =============================================================================
|
|
230
|
+
# CONVERSIONS SSE <-> JSON
|
|
231
|
+
# =============================================================================
|
|
232
|
+
def sse_to_message(sse_text: str) -> Tuple[Optional[dict], Optional[str], Optional[str]]:
|
|
233
|
+
"""(message, model, finish_reason) a partir d'un flux SSE OpenAI-compatible."""
|
|
234
|
+
content_parts: list = []
|
|
235
|
+
tool_calls: Dict[int, dict] = {}
|
|
236
|
+
model = None
|
|
237
|
+
finish = None
|
|
238
|
+
for raw_line in sse_text.split("\n"):
|
|
239
|
+
line = raw_line.strip()
|
|
240
|
+
if not line.startswith("data:") or line.startswith("data: [DONE]"):
|
|
241
|
+
continue
|
|
242
|
+
try:
|
|
243
|
+
chunk = json.loads(line[5:].strip())
|
|
244
|
+
except Exception:
|
|
245
|
+
continue
|
|
246
|
+
model = chunk.get("model") or model
|
|
247
|
+
for ch in (chunk.get("choices") or []):
|
|
248
|
+
delta = ch.get("delta") or {}
|
|
249
|
+
if delta.get("content"):
|
|
250
|
+
content_parts.append(delta["content"])
|
|
251
|
+
for tc in (delta.get("tool_calls") or []):
|
|
252
|
+
slot = tool_calls.setdefault(tc.get("index", 0), {"id": None, "type": "function",
|
|
253
|
+
"function": {"name": "", "arguments": ""}})
|
|
254
|
+
if tc.get("id"):
|
|
255
|
+
slot["id"] = tc["id"]
|
|
256
|
+
fn = tc.get("function") or {}
|
|
257
|
+
if fn.get("name"):
|
|
258
|
+
slot["function"]["name"] += fn["name"]
|
|
259
|
+
if fn.get("arguments"):
|
|
260
|
+
slot["function"]["arguments"] += fn["arguments"]
|
|
261
|
+
if ch.get("finish_reason"):
|
|
262
|
+
finish = ch["finish_reason"]
|
|
263
|
+
msg: Dict[str, Any] = {"role": "assistant", "content": "".join(content_parts) if content_parts else None}
|
|
264
|
+
if tool_calls:
|
|
265
|
+
msg["tool_calls"] = [tool_calls[i] for i in sorted(tool_calls)]
|
|
266
|
+
return msg, model, finish
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def message_to_sse(message: dict, model: Optional[str], finish: str = "stop") -> str:
|
|
270
|
+
delta: Dict[str, Any] = {"role": "assistant"}
|
|
271
|
+
if message.get("content"):
|
|
272
|
+
delta["content"] = message["content"]
|
|
273
|
+
if message.get("tool_calls"):
|
|
274
|
+
delta["tool_calls"] = [{"index": i, **tc} for i, tc in enumerate(message["tool_calls"])]
|
|
275
|
+
chunk = {"id": "chatcmpl-cache-" + uuid.uuid4().hex[:8], "object": "chat.completion.chunk",
|
|
276
|
+
"created": int(time.time()), "model": model or "unknown",
|
|
277
|
+
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}]}
|
|
278
|
+
return f"data: {json.dumps(chunk, ensure_ascii=False)}\n\ndata: [DONE]\n\n"
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def openai_usage(u: Dict[str, Any]) -> Dict[str, Any]:
|
|
282
|
+
pt = u.get("prompt_tokens")
|
|
283
|
+
ct = u.get("completion_tokens")
|
|
284
|
+
out = {"prompt_tokens": pt, "completion_tokens": ct}
|
|
285
|
+
if pt is not None or ct is not None:
|
|
286
|
+
out["total_tokens"] = (pt or 0) + (ct or 0)
|
|
287
|
+
return out
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def anthropic_usage(u: Dict[str, Any]) -> Dict[str, Any]:
|
|
291
|
+
"""Usage au format Anthropic natif (a partir de l'usage brut de l'upstream)."""
|
|
292
|
+
return {
|
|
293
|
+
"input_tokens": u.get("input_tokens"),
|
|
294
|
+
"output_tokens": u.get("output_tokens"),
|
|
295
|
+
"cache_read_input_tokens": u.get("cache_read_input_tokens") or 0,
|
|
296
|
+
"cache_creation_input_tokens": u.get("cache_creation_input_tokens") or 0,
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def anthropic_usage_from_norm(norm: Dict[str, Any]) -> Dict[str, Any]:
|
|
301
|
+
"""Usage Anthropic reconstruit depuis la forme normalisee (cache hit)."""
|
|
302
|
+
return {
|
|
303
|
+
"input_tokens": norm.get("prompt_tokens"),
|
|
304
|
+
"output_tokens": norm.get("completion_tokens"),
|
|
305
|
+
"cache_read_input_tokens": norm.get("cached_tokens") or 0,
|
|
306
|
+
"cache_creation_input_tokens": norm.get("cache_write_tokens") or 0,
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def build_openai_response(message: Optional[dict], model: Optional[str], finish: str,
|
|
311
|
+
usage: Dict[str, Any], cached: bool = False, extra: Optional[dict] = None) -> dict:
|
|
312
|
+
out = {
|
|
313
|
+
"id": "chatcmpl-" + uuid.uuid4().hex[:12],
|
|
314
|
+
"object": "chat.completion",
|
|
315
|
+
"created": int(time.time()),
|
|
316
|
+
"model": model or "unknown",
|
|
317
|
+
"choices": [{"index": 0, "message": message or {"role": "assistant", "content": ""},
|
|
318
|
+
"finish_reason": finish}],
|
|
319
|
+
"usage": usage,
|
|
320
|
+
}
|
|
321
|
+
if extra:
|
|
322
|
+
out.update(extra)
|
|
323
|
+
return out
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def build_anthropic_response(message: Optional[dict], model: Optional[str], stop: str,
|
|
327
|
+
usage: Dict[str, Any], extra: Optional[dict] = None) -> dict:
|
|
328
|
+
content = message.get("content") if isinstance(message, dict) else None
|
|
329
|
+
if isinstance(content, str):
|
|
330
|
+
content = [{"type": "text", "text": content}]
|
|
331
|
+
if not content:
|
|
332
|
+
content = []
|
|
333
|
+
out = {
|
|
334
|
+
"id": "msg_" + uuid.uuid4().hex[:16],
|
|
335
|
+
"type": "message",
|
|
336
|
+
"role": "assistant",
|
|
337
|
+
"model": model or "unknown",
|
|
338
|
+
"content": content,
|
|
339
|
+
"stop_reason": stop or "end_turn",
|
|
340
|
+
"stop_sequence": None,
|
|
341
|
+
"usage": usage,
|
|
342
|
+
}
|
|
343
|
+
if extra:
|
|
344
|
+
out.update(extra)
|
|
345
|
+
return out
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
# =============================================================================
|
|
349
|
+
# APPLICATION
|
|
350
|
+
# =============================================================================
|
|
351
|
+
@asynccontextmanager
|
|
352
|
+
async def lifespan(_app: FastAPI):
|
|
353
|
+
hub.bind(asyncio.get_running_loop())
|
|
354
|
+
icy_core.init_db()
|
|
355
|
+
yield
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
app = FastAPI(title="IcyCache Gateway", version="2.0.0", lifespan=lifespan)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _cache_age(created_at: Optional[str]) -> Optional[int]:
|
|
362
|
+
if not created_at:
|
|
363
|
+
return None
|
|
364
|
+
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S"):
|
|
365
|
+
try:
|
|
366
|
+
dt = datetime.strptime(str(created_at).split(".")[0], "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
|
|
367
|
+
return int((datetime.now(timezone.utc) - dt).total_seconds())
|
|
368
|
+
except Exception:
|
|
369
|
+
continue
|
|
370
|
+
return None
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
async def _read_body(request: Request) -> Optional[Dict[str, Any]]:
|
|
374
|
+
raw = await request.body()
|
|
375
|
+
if not raw:
|
|
376
|
+
return None
|
|
377
|
+
try:
|
|
378
|
+
data = json.loads(raw)
|
|
379
|
+
except Exception:
|
|
380
|
+
return None
|
|
381
|
+
return data if isinstance(data, dict) else None
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
@app.post("/v1/chat/completions")
|
|
385
|
+
@app.post("/v1/messages")
|
|
386
|
+
@app.post("/v1/embeddings")
|
|
387
|
+
async def llm_endpoint(request: Request, api: str = None):
|
|
388
|
+
"""Point d'entree LLM unifie. Le dispatch se fait sur le chemin reel."""
|
|
389
|
+
path = request.url.path
|
|
390
|
+
return await handle_llm(request, path)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
@app.api_route("/{upstream}/v1/{rest:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
|
|
394
|
+
async def prefixed_endpoint(upstream: str, rest: str, request: Request):
|
|
395
|
+
"""Routage explicite par prefixe : /omniroute/v1/chat/completions."""
|
|
396
|
+
return await handle_llm(request, "/v1/" + rest, prefix=upstream)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
async def handle_llm(request: Request, path: str, prefix: Optional[str] = None):
|
|
400
|
+
body = await _read_body(request)
|
|
401
|
+
if body is None and request.method != "GET":
|
|
402
|
+
raise HTTPException(status_code=400, detail={"error": "invalid_json",
|
|
403
|
+
"message": "Body JSON invalide ou absent."})
|
|
404
|
+
body = body or {}
|
|
405
|
+
upstream, source = resolve_upstream(request, body, prefix)
|
|
406
|
+
kind = upstream_kind(upstream)
|
|
407
|
+
|
|
408
|
+
if path.startswith("/v1/messages"):
|
|
409
|
+
want_kind = "anthropic"
|
|
410
|
+
elif path.startswith("/v1/embeddings") or path.startswith("/v1/models"):
|
|
411
|
+
want_kind = "openai"
|
|
412
|
+
else:
|
|
413
|
+
want_kind = "openai"
|
|
414
|
+
if kind != want_kind and not UPSTREAMS[upstream].get("allow_cross_kind"):
|
|
415
|
+
raise HTTPException(
|
|
416
|
+
status_code=400,
|
|
417
|
+
detail={"error": "protocol_mismatch",
|
|
418
|
+
"message": (f"L'upstream '{upstream}' parle le protocole '{kind}' mais ce chemin "
|
|
419
|
+
f"attend '{want_kind}'. Utilisez le chemin correspondant ou un autre upstream."),
|
|
420
|
+
"available_upstreams": _upstream_names()},
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
if path.startswith("/v1/models") and request.method == "GET":
|
|
424
|
+
return await proxy_models(request, upstream)
|
|
425
|
+
|
|
426
|
+
return await run_llm(request, body, path, upstream, kind, source)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
async def proxy_models(request: Request, upstream: str):
|
|
430
|
+
"""Liste des modeles de l'upstream (cache 60s)."""
|
|
431
|
+
url = target_url(upstream, "/v1/models")
|
|
432
|
+
try:
|
|
433
|
+
async with httpx.AsyncClient(timeout=30) as client:
|
|
434
|
+
resp = await client.get(url, headers=forward_headers(request, upstream, upstream_kind(upstream)))
|
|
435
|
+
if resp.status_code >= 400:
|
|
436
|
+
return Response(content=resp.content, status_code=resp.status_code,
|
|
437
|
+
media_type=resp.headers.get("content-type", "application/json"))
|
|
438
|
+
return Response(content=resp.content, media_type="application/json",
|
|
439
|
+
headers={"X-IcyCache-Upstream": upstream})
|
|
440
|
+
except httpx.RequestError as exc:
|
|
441
|
+
return JSONResponse({"error": {"message": f"upstream unreachable: {exc}", "type": "upstream_error"}},
|
|
442
|
+
status_code=502)
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
async def run_llm(request: Request, body: Dict[str, Any], path: str, upstream: str, kind: str, source: str):
|
|
446
|
+
t0 = time.perf_counter()
|
|
447
|
+
req_id = uuid.uuid4().hex[:12]
|
|
448
|
+
lab = client_label(request)
|
|
449
|
+
is_stream = bool(body.get("stream"))
|
|
450
|
+
|
|
451
|
+
# --- loop guard LLM (historique de tool calls identiques + resultats identiques) ---
|
|
452
|
+
_lg = CONFIG.get("loop_guard") or {}
|
|
453
|
+
if _lg.get("enabled", True) and isinstance(body.get("messages"), list):
|
|
454
|
+
_llm = _lg.get("llm") or {}
|
|
455
|
+
_loop = icy_core.llm_loop_check(body["messages"], warn_after=int(_llm.get("warn_after", 3)))
|
|
456
|
+
if _loop:
|
|
457
|
+
icy_core.log_agent_event("llm_loop", client=lab, detail=_loop)
|
|
458
|
+
if _llm.get("action") == "block" and _loop["reps"] >= int(_llm.get("block_after", 6)):
|
|
459
|
+
return JSONResponse({"error": {"type": "loop_detected", "loop": _loop,
|
|
460
|
+
"message": "Boucle d'agent detectee (memes tool calls + memes resultats)."}},
|
|
461
|
+
status_code=429)
|
|
462
|
+
|
|
463
|
+
# --- D2 : injection cache_control / prompt_cache_key selon l'upstream ---
|
|
464
|
+
out_body = json.loads(json.dumps(body)) # copie profonde (le body original n'est pas touche)
|
|
465
|
+
out_body.pop("upstream", None)
|
|
466
|
+
injected = mutate(kind, upstream, out_body)
|
|
467
|
+
|
|
468
|
+
cacheable, cache_reason = cacheable_reason(out_body)
|
|
469
|
+
ckey, raw_hash = build_cache_key(out_body, upstream, path)
|
|
470
|
+
is_chat = path.startswith("/v1/chat/completions") or path.startswith("/v1/messages")
|
|
471
|
+
is_embed = path.startswith("/v1/embeddings")
|
|
472
|
+
|
|
473
|
+
# --- HIT ---
|
|
474
|
+
if cacheable and (is_chat or is_embed):
|
|
475
|
+
hit = CACHE.get(ckey)
|
|
476
|
+
if hit:
|
|
477
|
+
return serve_hit(hit, body, path, upstream, kind, ckey, raw_hash, t0, lab, req_id)
|
|
478
|
+
|
|
479
|
+
# --- MISS ---
|
|
480
|
+
url = target_url(upstream, path)
|
|
481
|
+
headers = forward_headers(request, upstream, kind)
|
|
482
|
+
if is_stream and kind == "openai":
|
|
483
|
+
out_body["stream_options"] = {**(out_body.get("stream_options") or {}), "include_usage": True}
|
|
484
|
+
|
|
485
|
+
if is_stream:
|
|
486
|
+
return await forward_stream(req_id, url, headers, out_body, upstream, kind, ckey, raw_hash,
|
|
487
|
+
cacheable, cache_reason, t0, lab, path, is_chat, is_embed, injected)
|
|
488
|
+
return await forward_json(req_id, url, headers, out_body, upstream, kind, ckey, raw_hash,
|
|
489
|
+
cacheable, cache_reason, t0, lab, path, is_chat, is_embed, injected)
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def serve_hit(hit: Dict[str, Any], body: Dict[str, Any], path: str, upstream: str, kind: str,
|
|
493
|
+
ckey: str, raw_hash: str, t0: float, lab: str, req_id: str) -> Response:
|
|
494
|
+
"""Reponse depuis le cache : tokens sauvegardes = usage reel du premier appel."""
|
|
495
|
+
entry = decode_entry(hit["response"])
|
|
496
|
+
usage = entry.get("usage") or {}
|
|
497
|
+
pt = usage.get("prompt_tokens")
|
|
498
|
+
ct = usage.get("completion_tokens")
|
|
499
|
+
model = entry.get("model") or hit.get("model")
|
|
500
|
+
finish = entry.get("finish_reason") or "stop"
|
|
501
|
+
latency_ms = int((time.perf_counter() - t0) * 1000)
|
|
502
|
+
|
|
503
|
+
if kind == "anthropic":
|
|
504
|
+
sp = usage.get("prompt_tokens")
|
|
505
|
+
sc = usage.get("completion_tokens")
|
|
506
|
+
log_event("HIT", ckey, body.get("model"), model, bool(body.get("stream")),
|
|
507
|
+
sp=sp, sc=sc, latency_ms=latency_ms, upstream=upstream, client=lab,
|
|
508
|
+
cached_tokens=usage.get("cached_tokens"), req_id=req_id)
|
|
509
|
+
if body.get("stream"):
|
|
510
|
+
return _replay_anthropic_stream(entry, model, finish, usage)
|
|
511
|
+
return JSONResponse(build_anthropic_response(
|
|
512
|
+
entry.get("message"), model, finish, anthropic_usage_from_norm(usage),
|
|
513
|
+
extra={"cached": True, "cache_hash": ckey[:8],
|
|
514
|
+
"saved_prompt_tokens": sp, "saved_completion_tokens": sc,
|
|
515
|
+
"cache_age_seconds": _cache_age(hit.get("created_at")),
|
|
516
|
+
"hit_count": hit.get("access_count", 1),
|
|
517
|
+
"original_usage": anthropic_usage_from_norm(usage)}))
|
|
518
|
+
|
|
519
|
+
sp = pt or 0
|
|
520
|
+
sc = ct or 0
|
|
521
|
+
log_event("HIT", ckey, body.get("model"), model, bool(body.get("stream")),
|
|
522
|
+
sp=sp, sc=sc, latency_ms=latency_ms, upstream=upstream, client=lab,
|
|
523
|
+
cached_tokens=usage.get("cached_tokens"), req_id=req_id)
|
|
524
|
+
|
|
525
|
+
if body.get("stream"):
|
|
526
|
+
sse = entry.get("sse")
|
|
527
|
+
if not sse:
|
|
528
|
+
sse = message_to_sse(entry.get("message") or {}, model, finish)
|
|
529
|
+
return StreamingResponse(_sse_iter(sse), media_type="text/event-stream",
|
|
530
|
+
headers={"X-IcyCache-Upstream": upstream, "X-IcyCache-Hit": "1"})
|
|
531
|
+
|
|
532
|
+
return JSONResponse(build_openai_response(
|
|
533
|
+
entry.get("message"), model, finish,
|
|
534
|
+
{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
|
|
535
|
+
extra={"cached": True, "cache_hash": ckey[:8],
|
|
536
|
+
"saved_prompt_tokens": sp, "saved_completion_tokens": sc,
|
|
537
|
+
"saved_total_tokens": sp + sc,
|
|
538
|
+
"original_gen_time_ms": entry.get("gen_time_ms"),
|
|
539
|
+
"cache_age_seconds": _cache_age(hit.get("created_at")),
|
|
540
|
+
"hit_count": hit.get("access_count", 1),
|
|
541
|
+
"original_usage": usage}),
|
|
542
|
+
headers={"X-IcyCache-Upstream": upstream, "X-IcyCache-Hit": "1"})
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
async def _sse_iter(text: str):
|
|
546
|
+
"""Rejoue un flux SSE stocke tel quel (separateurs \\n\\n preserves)."""
|
|
547
|
+
if not text:
|
|
548
|
+
return
|
|
549
|
+
for block in text.split("\n\n"):
|
|
550
|
+
block = block.strip("\n")
|
|
551
|
+
if block:
|
|
552
|
+
yield block + "\n\n"
|
|
553
|
+
if "[DONE]" not in text:
|
|
554
|
+
yield "data: [DONE]\n\n"
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def _replay_anthropic_stream(entry: Dict[str, Any], model: Optional[str], finish: str,
|
|
558
|
+
usage: Dict[str, Any]) -> Response:
|
|
559
|
+
sse = entry.get("sse")
|
|
560
|
+
if not sse:
|
|
561
|
+
msg = entry.get("message") or {}
|
|
562
|
+
content = msg.get("content") if isinstance(msg, dict) else ""
|
|
563
|
+
blocks = [{"type": "text", "text": content}] if isinstance(content, str) else (content or [])
|
|
564
|
+
sse = (
|
|
565
|
+
"event: message_start\n"
|
|
566
|
+
"data: " + json.dumps({"type": "message_start", "message": {
|
|
567
|
+
"id": "msg_cache", "type": "message", "role": "assistant", "model": model,
|
|
568
|
+
"content": [], "stop_reason": None,
|
|
569
|
+
"usage": {"input_tokens": usage.get("prompt_tokens") or 0,
|
|
570
|
+
"output_tokens": 0}}}, ensure_ascii=False) + "\n\n"
|
|
571
|
+
"event: content_block_delta\n"
|
|
572
|
+
"data: " + json.dumps({"type": "content_block_delta", "index": 0,
|
|
573
|
+
"delta": {"type": "text_delta", "text": ""}}, ensure_ascii=False) + "\n\n"
|
|
574
|
+
"event: message_delta\n"
|
|
575
|
+
"data: " + json.dumps({"type": "message_delta",
|
|
576
|
+
"usage": {"output_tokens": usage.get("completion_tokens") or 0}},
|
|
577
|
+
ensure_ascii=False) + "\n\n"
|
|
578
|
+
"event: message_stop\n"
|
|
579
|
+
"data: " + json.dumps({"type": "message_stop"}, ensure_ascii=False) + "\n\n"
|
|
580
|
+
)
|
|
581
|
+
_ = blocks
|
|
582
|
+
return StreamingResponse(_sse_iter(sse), media_type="text/event-stream",
|
|
583
|
+
headers={"X-IcyCache-Upstream": "cache", "X-IcyCache-Hit": "1"})
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
async def forward_json(req_id: str, url: str, headers: Dict[str, str], body: Dict[str, Any],
|
|
587
|
+
upstream: str, kind: str, ckey: str, raw_hash: str, cacheable: bool,
|
|
588
|
+
cache_reason: str, t0: float, lab: str, path: str, is_chat: bool,
|
|
589
|
+
is_embed: bool, injected: Optional[str]):
|
|
590
|
+
t_fwd = time.perf_counter()
|
|
591
|
+
try:
|
|
592
|
+
async with httpx.AsyncClient(timeout=TIMEOUT) as client:
|
|
593
|
+
resp = await client.post(url, json=body, headers=headers)
|
|
594
|
+
except httpx.RequestError as exc:
|
|
595
|
+
return JSONResponse({"error": {"message": f"upstream unreachable: {exc}", "type": "upstream_error",
|
|
596
|
+
"upstream": upstream}}, status_code=502)
|
|
597
|
+
|
|
598
|
+
if resp.status_code >= 400:
|
|
599
|
+
return Response(content=resp.content, status_code=resp.status_code,
|
|
600
|
+
media_type=resp.headers.get("content-type", "application/json"),
|
|
601
|
+
headers={"X-IcyCache-Upstream": upstream})
|
|
602
|
+
|
|
603
|
+
gen_time = int((time.perf_counter() - t_fwd) * 1000)
|
|
604
|
+
latency_ms = int((time.perf_counter() - t0) * 1000)
|
|
605
|
+
try:
|
|
606
|
+
data = resp.json()
|
|
607
|
+
except Exception:
|
|
608
|
+
data = {}
|
|
609
|
+
|
|
610
|
+
raw_usage = data.get("usage") or {}
|
|
611
|
+
norm = usage_from(kind, raw_usage)
|
|
612
|
+
pt = norm.get("prompt_tokens")
|
|
613
|
+
ct = norm.get("completion_tokens")
|
|
614
|
+
model = data.get("model") or body.get("model")
|
|
615
|
+
|
|
616
|
+
message = None
|
|
617
|
+
finish = "stop"
|
|
618
|
+
if kind == "anthropic":
|
|
619
|
+
content = data.get("content")
|
|
620
|
+
message = {"role": "assistant", "content": content if content is not None else ""}
|
|
621
|
+
finish = data.get("stop_reason") or "end_turn"
|
|
622
|
+
else:
|
|
623
|
+
choices = data.get("choices") or []
|
|
624
|
+
if choices:
|
|
625
|
+
message = choices[0].get("message") or {}
|
|
626
|
+
finish = choices[0].get("finish_reason") or "stop"
|
|
627
|
+
|
|
628
|
+
log_event("MISS" if cacheable else "BYPASS", ckey, body.get("model"), model, False, pt, ct,
|
|
629
|
+
latency_ms=latency_ms, upstream=upstream, client=lab,
|
|
630
|
+
cached_tokens=norm.get("cached_tokens"),
|
|
631
|
+
cache_write_tokens=norm.get("cache_write_tokens"), req_id=req_id)
|
|
632
|
+
|
|
633
|
+
if cacheable and (is_chat or is_embed):
|
|
634
|
+
stored = encode_entry(kind, message, norm, model, finish, "", gen_time)
|
|
635
|
+
try:
|
|
636
|
+
CACHE.set(ckey, icy_normalize.get_default_normalizer().normalize(display_prompt(body)),
|
|
637
|
+
display_prompt(body), stored, False, upstream, model, raw_hash)
|
|
638
|
+
except Exception:
|
|
639
|
+
_log.exception("CACHE.set (json) a echoue")
|
|
640
|
+
|
|
641
|
+
if kind == "anthropic":
|
|
642
|
+
payload = build_anthropic_response(message, model, finish, anthropic_usage(raw_usage))
|
|
643
|
+
else:
|
|
644
|
+
payload = data if isinstance(data, dict) and data else build_openai_response(message, model, finish,
|
|
645
|
+
openai_usage(norm))
|
|
646
|
+
return JSONResponse(payload, headers={"X-IcyCache-Upstream": upstream,
|
|
647
|
+
"X-IcyCache-Tokens-Source": "upstream" if pt is not None else "n/a"})
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
async def forward_stream(req_id: str, url: str, headers: Dict[str, str], body: Dict[str, Any],
|
|
651
|
+
upstream: str, kind: str, ckey: str, raw_hash: str, cacheable: bool,
|
|
652
|
+
cache_reason: str, t0: float, lab: str, path: str, is_chat: bool,
|
|
653
|
+
is_embed: bool, injected: Optional[str]):
|
|
654
|
+
t_fwd = time.perf_counter()
|
|
655
|
+
client = httpx.AsyncClient(timeout=TIMEOUT)
|
|
656
|
+
try:
|
|
657
|
+
response = await client.send(client.build_request("POST", url, json=body, headers=headers), stream=True)
|
|
658
|
+
except httpx.RequestError as exc:
|
|
659
|
+
await client.aclose()
|
|
660
|
+
return JSONResponse({"error": {"message": f"upstream unreachable: {exc}", "type": "upstream_error",
|
|
661
|
+
"upstream": upstream}}, status_code=502)
|
|
662
|
+
|
|
663
|
+
if response.status_code >= 400:
|
|
664
|
+
err = await response.aread()
|
|
665
|
+
ctype = response.headers.get("content-type", "application/json")
|
|
666
|
+
status = response.status_code
|
|
667
|
+
await response.aclose()
|
|
668
|
+
await client.aclose()
|
|
669
|
+
return Response(content=err, status_code=status, media_type=ctype,
|
|
670
|
+
headers={"X-IcyCache-Upstream": upstream})
|
|
671
|
+
|
|
672
|
+
meter = StreamMeter(kind)
|
|
673
|
+
state = {"chunks": 0, "ttft_ms": None, "sse": [], "last_push": 0.0, "ok": False}
|
|
674
|
+
|
|
675
|
+
async def gen():
|
|
676
|
+
"""Rejoue les bytes upstream tels quels (fidelite SSE), tout en mesurant.
|
|
677
|
+
|
|
678
|
+
- les octets recus sont forwarding intacts (le client voit exactement
|
|
679
|
+
ce que l'upstream a produit) ;
|
|
680
|
+
- les lignes completes sont en parallele passees au StreamMeter pour
|
|
681
|
+
extraire l'usage REEL et le modele ;
|
|
682
|
+
- le compteur de chunks et le TTFT alimentent le WebSocket.
|
|
683
|
+
"""
|
|
684
|
+
buffer = b""
|
|
685
|
+
try:
|
|
686
|
+
async for raw in response.aiter_bytes():
|
|
687
|
+
buffer += raw
|
|
688
|
+
while b"\n" in buffer:
|
|
689
|
+
line_bytes, buffer = buffer.split(b"\n", 1)
|
|
690
|
+
meter.feed(line_bytes + b"\n")
|
|
691
|
+
line = line_bytes.decode("utf-8", "replace").strip()
|
|
692
|
+
if state["ttft_ms"] is None and line.startswith("data:"):
|
|
693
|
+
try:
|
|
694
|
+
ev = json.loads(line[5:].strip())
|
|
695
|
+
delta = (ev.get("choices") or [{}])[0].get("delta") or {}
|
|
696
|
+
if delta.get("content") or ev.get("type") == "content_block_delta":
|
|
697
|
+
state["ttft_ms"] = int((time.perf_counter() - t_fwd) * 1000)
|
|
698
|
+
except Exception:
|
|
699
|
+
pass
|
|
700
|
+
state["chunks"] += 1
|
|
701
|
+
state["sse"].append(raw.decode("utf-8", "replace"))
|
|
702
|
+
now = time.perf_counter()
|
|
703
|
+
if now - state["last_push"] >= WS_DELTA_INTERVAL:
|
|
704
|
+
state["last_push"] = now
|
|
705
|
+
hub.publish({"type": "delta", "id": req_id, "ts": utc_iso(), "upstream": upstream,
|
|
706
|
+
"client": lab, "model": meter.model or body.get("model"),
|
|
707
|
+
"chunks": state["chunks"], "ttft_ms": state["ttft_ms"]})
|
|
708
|
+
yield raw
|
|
709
|
+
if buffer:
|
|
710
|
+
meter.feed(buffer)
|
|
711
|
+
state["ok"] = True
|
|
712
|
+
except (asyncio.CancelledError, GeneratorExit):
|
|
713
|
+
raise
|
|
714
|
+
except Exception:
|
|
715
|
+
pass
|
|
716
|
+
finally:
|
|
717
|
+
try:
|
|
718
|
+
await response.aclose()
|
|
719
|
+
await client.aclose()
|
|
720
|
+
except Exception:
|
|
721
|
+
pass
|
|
722
|
+
|
|
723
|
+
norm = usage_from(kind, meter.usage)
|
|
724
|
+
pt = norm.get("prompt_tokens")
|
|
725
|
+
ct = norm.get("completion_tokens")
|
|
726
|
+
latency_ms = int((time.perf_counter() - t_fwd) * 1000)
|
|
727
|
+
model = meter.model or body.get("model")
|
|
728
|
+
log_event("MISS" if cacheable else "BYPASS", ckey, body.get("model"), model, True, pt, ct,
|
|
729
|
+
latency_ms=latency_ms, ttft_ms=state["ttft_ms"], upstream=upstream, client=lab,
|
|
730
|
+
cached_tokens=norm.get("cached_tokens"),
|
|
731
|
+
cache_write_tokens=norm.get("cache_write_tokens"), req_id=req_id)
|
|
732
|
+
|
|
733
|
+
_sse_all = "".join(state["sse"])
|
|
734
|
+
_complete = state["ok"] and (("[DONE]" in _sse_all) if kind == "openai"
|
|
735
|
+
else ("message_stop" in _sse_all))
|
|
736
|
+
if cacheable and (is_chat or is_embed) and not _complete:
|
|
737
|
+
_log.warning("flux incomplet: NON cache (req %s, upstream %s)", req_id, upstream)
|
|
738
|
+
if cacheable and (is_chat or is_embed) and _complete:
|
|
739
|
+
sse_text = _sse_all
|
|
740
|
+
if kind == "openai":
|
|
741
|
+
msg, m2, finish = sse_to_message(sse_text)
|
|
742
|
+
stored = encode_entry("openai", msg, norm, m2 or model, finish or "stop", sse_text, latency_ms)
|
|
743
|
+
else:
|
|
744
|
+
stored = encode_entry("anthropic", None, norm, model, "end_turn", sse_text, latency_ms)
|
|
745
|
+
try:
|
|
746
|
+
CACHE.set(ckey, icy_normalize.get_default_normalizer().normalize(display_prompt(body)),
|
|
747
|
+
display_prompt(body), stored, True, upstream, model, raw_hash)
|
|
748
|
+
except Exception:
|
|
749
|
+
_log.exception("CACHE.set (stream) a echoue")
|
|
750
|
+
|
|
751
|
+
return StreamingResponse(gen(), media_type="text/event-stream",
|
|
752
|
+
headers={"X-IcyCache-Upstream": upstream,
|
|
753
|
+
"X-IcyCache-Cache-Reason": cache_reason,
|
|
754
|
+
"X-IcyCache-Injected": injected or "none"})
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
# =============================================================================
|
|
758
|
+
# WEBSOCKET /ws/events
|
|
759
|
+
# =============================================================================
|
|
760
|
+
@app.websocket("/ws/events")
|
|
761
|
+
async def ws_events(websocket: WebSocket):
|
|
762
|
+
"""Flux temps reel : 'delta' pendant le streaming, 'final' a la fin (tokens reels)."""
|
|
763
|
+
origin = websocket.headers.get("origin")
|
|
764
|
+
if origin and not re.match(r"^https?://(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$", origin):
|
|
765
|
+
await websocket.close(code=1008)
|
|
766
|
+
return
|
|
767
|
+
await websocket.accept()
|
|
768
|
+
queue = hub.subscribe()
|
|
769
|
+
subs = {"upstreams": None}
|
|
770
|
+
|
|
771
|
+
async def pump():
|
|
772
|
+
try:
|
|
773
|
+
while True:
|
|
774
|
+
ev = await queue.get()
|
|
775
|
+
if subs["upstreams"] and ev.get("upstream") not in subs["upstreams"]:
|
|
776
|
+
continue
|
|
777
|
+
await websocket.send_json(ev)
|
|
778
|
+
except Exception:
|
|
779
|
+
return
|
|
780
|
+
|
|
781
|
+
pump_task = asyncio.create_task(pump())
|
|
782
|
+
try:
|
|
783
|
+
await websocket.send_json({
|
|
784
|
+
"type": "hello",
|
|
785
|
+
"ts": utc_iso(),
|
|
786
|
+
"protocol": "icycache.ws/v1",
|
|
787
|
+
"upstreams": _upstream_names(),
|
|
788
|
+
"default_upstream": DEFAULT_UPSTREAM,
|
|
789
|
+
"note": "Tokens = usage reel de l'upstream. NULL = absent chez l'upstream (jamais estime).",
|
|
790
|
+
})
|
|
791
|
+
while True:
|
|
792
|
+
raw = await websocket.receive_text()
|
|
793
|
+
try:
|
|
794
|
+
msg = json.loads(raw)
|
|
795
|
+
except Exception:
|
|
796
|
+
continue
|
|
797
|
+
mtype = msg.get("type")
|
|
798
|
+
if mtype == "ping":
|
|
799
|
+
await websocket.send_json({"type": "pong", "ts": utc_iso()})
|
|
800
|
+
elif mtype == "subscribe":
|
|
801
|
+
subs["upstreams"] = set(msg.get("upstreams") or []) or None
|
|
802
|
+
await websocket.send_json({"type": "subscribed", "ts": utc_iso(),
|
|
803
|
+
"upstreams": sorted(subs["upstreams"]) if subs["upstreams"] else None})
|
|
804
|
+
except WebSocketDisconnect:
|
|
805
|
+
pass
|
|
806
|
+
except Exception:
|
|
807
|
+
pass
|
|
808
|
+
finally:
|
|
809
|
+
pump_task.cancel()
|
|
810
|
+
hub.unsubscribe(queue)
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
# =============================================================================
|
|
814
|
+
# OBSERVABILITE (formes compatibles avec le dashboard existant)
|
|
815
|
+
# =============================================================================
|
|
816
|
+
def _tcp_alive(host: str, port: int, timeout: float = 0.7) -> Tuple[bool, Optional[float]]:
|
|
817
|
+
t0 = time.perf_counter()
|
|
818
|
+
try:
|
|
819
|
+
with socket.create_connection((host, int(port)), timeout=timeout):
|
|
820
|
+
return True, round((time.perf_counter() - t0) * 1000, 1)
|
|
821
|
+
except OSError:
|
|
822
|
+
return False, None
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
@app.get("/health")
|
|
826
|
+
async def health_check():
|
|
827
|
+
return {"status": "healthy", "service": "IcyCache", "project_root": str(PROJECT_ROOT),
|
|
828
|
+
"cache_stats": CACHE.stats(),
|
|
829
|
+
"upstreams": _upstream_names(), "default_upstream": DEFAULT_UPSTREAM}
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
@app.get("/api/status")
|
|
833
|
+
async def api_status():
|
|
834
|
+
mcp_host = CONFIG.get("mcp", {}).get("host", "127.0.0.1")
|
|
835
|
+
mcp_port = int(CONFIG.get("mcp", {}).get("port", 8081))
|
|
836
|
+
mcp_alive, mcp_ms = _tcp_alive(mcp_host, mcp_port)
|
|
837
|
+
ups = {}
|
|
838
|
+
for name, cfg in UPSTREAMS.items():
|
|
839
|
+
m = re.search(r"://([^:/]+)(?::(\d+))?", str(cfg.get("base_url", "")))
|
|
840
|
+
if m:
|
|
841
|
+
host = m.group(1)
|
|
842
|
+
port = int(m.group(2) or (443 if str(cfg.get("base_url", "")).startswith("https") else 80))
|
|
843
|
+
alive, ms = _tcp_alive(host, port)
|
|
844
|
+
ups[name] = {"alive": alive, "latency_ms": ms, "base_url": cfg.get("base_url"),
|
|
845
|
+
"kind": cfg.get("kind", "openai")}
|
|
846
|
+
return {
|
|
847
|
+
"proxy": {"alive": True, "host": PROXY_CFG.get("host", "127.0.0.1"),
|
|
848
|
+
"port": PROXY_CFG.get("port", 8080)},
|
|
849
|
+
"mcp": {"alive": mcp_alive, "latency_ms": mcp_ms, "host": mcp_host, "port": mcp_port,
|
|
850
|
+
"transports": CONFIG.get("mcp", {}).get("transport", []), "tools": list(_MCP_TOOL_NAMES)},
|
|
851
|
+
"upstream": ups,
|
|
852
|
+
"gateway": {"default_upstream": DEFAULT_UPSTREAM, "upstreams": _upstream_names(),
|
|
853
|
+
"websocket": "/ws/events"},
|
|
854
|
+
"server_time": datetime.now(timezone.utc).isoformat(),
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
@app.get("/api/dashboard-stats")
|
|
859
|
+
async def dashboard_stats():
|
|
860
|
+
import sqlite3
|
|
861
|
+
conn = sqlite3.connect(DB_FILE)
|
|
862
|
+
try:
|
|
863
|
+
rows = conn.execute("SELECT prompt_hash, model, normalized_prompt, access_count, "
|
|
864
|
+
"datetime(created_at, 'localtime') FROM cache ORDER BY last_accessed DESC LIMIT 50").fetchall()
|
|
865
|
+
finally:
|
|
866
|
+
conn.close()
|
|
867
|
+
prompts = [{"hash": r[0][:8] + "...", "model": r[1],
|
|
868
|
+
"normalized_prompt": (r[2] or "")[:80] + ("..." if len(r[2] or "") > 80 else ""),
|
|
869
|
+
"access_count": r[3], "created_at": r[4]} for r in rows]
|
|
870
|
+
# no-store: sans cet en-tete le navigateur ressert un payload perime et le
|
|
871
|
+
# dashboard affiche des compteurs figes (0) alors que la base a des donnees.
|
|
872
|
+
return JSONResponse({"stats": CACHE.stats(), "target_provider": DEFAULT_UPSTREAM, "prompts": prompts,
|
|
873
|
+
"upstreams": _upstream_names()},
|
|
874
|
+
headers={"Cache-Control": "no-store, must-revalidate"})
|
|
875
|
+
|
|
876
|
+
|
|
877
|
+
@app.get("/api/metrics")
|
|
878
|
+
async def api_metrics():
|
|
879
|
+
import sqlite3
|
|
880
|
+
conn = sqlite3.connect(DB_FILE)
|
|
881
|
+
try:
|
|
882
|
+
ok = "kind!='HIT' AND completion_tokens>0 AND latency_ms>0"
|
|
883
|
+
r = conn.execute(f"""SELECT COUNT(*), COALESCE(SUM(kind='HIT'),0), COALESCE(SUM(kind='MISS'),0),
|
|
884
|
+
COALESCE(SUM(kind='BYPASS'),0),
|
|
885
|
+
COALESCE(SUM(CASE WHEN kind!='HIT' THEN prompt_tokens END),0),
|
|
886
|
+
COALESCE(SUM(CASE WHEN kind!='HIT' THEN completion_tokens END),0),
|
|
887
|
+
COALESCE(SUM(saved_prompt),0), COALESCE(SUM(saved_completion),0),
|
|
888
|
+
COALESCE(SUM(CASE WHEN {ok} THEN completion_tokens END),0),
|
|
889
|
+
COALESCE(SUM(CASE WHEN {ok} THEN latency_ms END),0),
|
|
890
|
+
COALESCE(SUM(CASE WHEN kind!='HIT' AND prompt_tokens IS NULL THEN 1 END),0),
|
|
891
|
+
COALESCE(SUM(CASE WHEN kind!='HIT' AND completion_tokens>0 AND ttft_ms>0 AND latency_ms>ttft_ms+500 THEN completion_tokens END),0),
|
|
892
|
+
COALESCE(SUM(CASE WHEN kind!='HIT' AND completion_tokens>0 AND ttft_ms>0 AND latency_ms>ttft_ms+500 THEN latency_ms-ttft_ms END),0)
|
|
893
|
+
FROM events""").fetchone()
|
|
894
|
+
rows = conn.execute("""SELECT datetime(ts,'localtime'), kind, model_real, prompt_tokens,
|
|
895
|
+
completion_tokens, latency_ms, ttft_ms, saved_prompt+saved_completion
|
|
896
|
+
FROM events ORDER BY id DESC LIMIT 50""").fetchall()
|
|
897
|
+
finally:
|
|
898
|
+
conn.close()
|
|
899
|
+
n, hits, miss, byp, tin, tout, sin, sout, tk_c, tk_ms, no_usage, d_c, d_ms = r
|
|
900
|
+
spent, saved = tin + tout, sin + sout
|
|
901
|
+
recent = [{"ts": x[0], "kind": x[1], "model": x[2], "in": x[3], "out": x[4],
|
|
902
|
+
"tks": round(x[4] * 1000 / x[5], 1) if x[1] != "HIT" and x[4] and x[5] else None,
|
|
903
|
+
"tks_decode": round(x[4] * 1000 / (x[5] - x[6]), 1) if x[1] != "HIT" and x[4] and x[5] and x[6] and x[5] - x[6] >= 500 else None,
|
|
904
|
+
"latency_ms": x[5], "ttft_ms": x[6], "saved": x[7]} for x in rows]
|
|
905
|
+
return {"requests": n, "hits": hits, "misses": miss, "bypass": byp,
|
|
906
|
+
"hit_rate_pct": round(100 * hits / n, 1) if n else 0,
|
|
907
|
+
"tokens_in_spent": tin, "tokens_out_spent": tout,
|
|
908
|
+
"tokens_in_saved": sin, "tokens_out_saved": sout,
|
|
909
|
+
"pct_saved": round(100 * saved / (saved + spent), 1) if (saved + spent) else 0,
|
|
910
|
+
"tks_avg": round(tk_c * 1000 / tk_ms, 1) if tk_ms else None,
|
|
911
|
+
"tks_decode_avg": round(d_c * 1000 / d_ms, 1) if d_ms else None,
|
|
912
|
+
"tks_decode_last": next((x["tks_decode"] for x in recent if x["tks_decode"]), None),
|
|
913
|
+
"tks_last": next((x["tks"] for x in recent if x["tks"]), None),
|
|
914
|
+
"events_without_usage": no_usage, "recent": recent}
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
@app.get("/api/metrics/series")
|
|
918
|
+
async def api_metrics_series(bucket: str = "hour", hours: int = 24):
|
|
919
|
+
import sqlite3
|
|
920
|
+
hours = max(1, min(int(hours), 720))
|
|
921
|
+
fmt = "%Y-%m-%d %H:%M" if bucket == "min" else "%Y-%m-%d %H:00"
|
|
922
|
+
bucket = "min" if bucket == "min" else "hour"
|
|
923
|
+
conn = sqlite3.connect(DB_FILE)
|
|
924
|
+
try:
|
|
925
|
+
pts = conn.execute(f"""SELECT strftime('{fmt}', ts, 'localtime') AS b,
|
|
926
|
+
COALESCE(SUM(kind='HIT'),0), COALESCE(SUM(kind='MISS'),0), COALESCE(SUM(kind='BYPASS'),0),
|
|
927
|
+
COALESCE(SUM(saved_prompt+saved_completion),0),
|
|
928
|
+
COALESCE(SUM(CASE WHEN kind!='HIT' THEN COALESCE(prompt_tokens,0)+COALESCE(completion_tokens,0) END),0)
|
|
929
|
+
FROM events WHERE ts >= datetime('now', '-' || ? || ' hours') GROUP BY b ORDER BY b""",
|
|
930
|
+
(hours,)).fetchall()
|
|
931
|
+
mdl = conn.execute("""SELECT COALESCE(model_real, model_req, '?'), COUNT(*),
|
|
932
|
+
COALESCE(SUM(kind='HIT'),0), COALESCE(SUM(saved_prompt+saved_completion),0),
|
|
933
|
+
COALESCE(SUM(CASE WHEN kind!='HIT' THEN COALESCE(prompt_tokens,0)+COALESCE(completion_tokens,0) END),0)
|
|
934
|
+
FROM events WHERE ts >= datetime('now', '-' || ? || ' hours') GROUP BY 1 ORDER BY 2 DESC LIMIT 10""",
|
|
935
|
+
(hours,)).fetchall()
|
|
936
|
+
finally:
|
|
937
|
+
conn.close()
|
|
938
|
+
return {"bucket": bucket, "hours": hours,
|
|
939
|
+
"points": [{"t": p[0], "hits": p[1], "misses": p[2], "bypass": p[3], "saved": p[4], "spent": p[5]} for p in pts],
|
|
940
|
+
"models": [{"model": m[0], "requests": m[1], "hits": m[2], "saved": m[3], "spent": m[4]} for m in mdl]}
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
@app.get("/api/metrics/export")
|
|
944
|
+
async def api_metrics_export(since: str = "", limit: int = 10000):
|
|
945
|
+
import sqlite3
|
|
946
|
+
limit = max(1, min(int(limit), 100000))
|
|
947
|
+
conn = sqlite3.connect(DB_FILE)
|
|
948
|
+
conn.row_factory = sqlite3.Row
|
|
949
|
+
try:
|
|
950
|
+
q = ("SELECT id, ts, kind, hash, model_req, model_real, stream, prompt_tokens, completion_tokens, "
|
|
951
|
+
"saved_prompt, saved_completion, latency_ms, ttft_ms, upstream, client, cached_tokens FROM events")
|
|
952
|
+
args = []
|
|
953
|
+
if since:
|
|
954
|
+
q += " WHERE ts >= ?"
|
|
955
|
+
args.append(since)
|
|
956
|
+
q += " ORDER BY id ASC LIMIT ?"
|
|
957
|
+
args.append(limit)
|
|
958
|
+
rows = [dict(r) for r in conn.execute(q, args).fetchall()]
|
|
959
|
+
finally:
|
|
960
|
+
conn.close()
|
|
961
|
+
sp = sum((r["prompt_tokens"] or 0) for r in rows if r["kind"] != "HIT")
|
|
962
|
+
so = sum((r["completion_tokens"] or 0) for r in rows if r["kind"] != "HIT")
|
|
963
|
+
vp = sum(r["saved_prompt"] or 0 for r in rows)
|
|
964
|
+
vo = sum(r["saved_completion"] or 0 for r in rows)
|
|
965
|
+
return {"schema": "icycache.metrics.export/v1",
|
|
966
|
+
"generated_at_utc": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"),
|
|
967
|
+
"since": since or None, "count": len(rows),
|
|
968
|
+
"summary": {"tokens_in_spent": sp, "tokens_out_spent": so, "tokens_in_saved": vp,
|
|
969
|
+
"tokens_out_saved": vo,
|
|
970
|
+
"pct_saved": round(100 * (vp + vo) / (vp + vo + sp + so), 1) if (vp + vo + sp + so) else 0},
|
|
971
|
+
"events": rows}
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
@app.post("/api/cache-clear")
|
|
975
|
+
async def api_clear_cache(request: Request):
|
|
976
|
+
# X-IcyCache-Confirm = naming unifie. X-CacheForge-Confirm reste accepte
|
|
977
|
+
# (le dashboard n'a pas encore ete renomme).
|
|
978
|
+
confirm = (request.headers.get("x-icycache-confirm")
|
|
979
|
+
or request.headers.get("x-cacheforge-confirm"))
|
|
980
|
+
if confirm != "FORCE_DELETE":
|
|
981
|
+
raise HTTPException(status_code=403, detail="Missing header X-IcyCache-Confirm: FORCE_DELETE")
|
|
982
|
+
n = CACHE.clear()
|
|
983
|
+
return {"message": f"Cache vidé ({n} entrées)", "success": True, "deleted": n}
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
# Outils exposes par la couche MCP (liste de mcp_server.py ; sert au compteur du dashboard).
|
|
987
|
+
_MCP_TOOL_NAMES = ("check_tool_loop", "clear_cache", "clear_tool_cache", "get_cache_stats",
|
|
988
|
+
"get_tool_cache_stats", "list_proxy_servers", "list_proxy_tools",
|
|
989
|
+
"normalize_preview", "proxy_tool", "proxy_tools", "search_cached_prompts")
|
|
990
|
+
|
|
991
|
+
|
|
992
|
+
@app.get("/api/tools")
|
|
993
|
+
async def api_list_tools():
|
|
994
|
+
"""Serveurs MCP routables via le proxy d'outils (config/icycache.json -> tool_proxy)."""
|
|
995
|
+
try:
|
|
996
|
+
import tool_proxy as tp
|
|
997
|
+
except Exception as exc:
|
|
998
|
+
return JSONResponse({"ok": False, "error": f"tool_proxy indisponible: {exc}"}, status_code=503)
|
|
999
|
+
names = tp.server_names()
|
|
1000
|
+
return {"ok": True, "servers": names, "count": len(names)}
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
@app.get("/api/tools/stats")
|
|
1004
|
+
async def api_tool_cache_stats():
|
|
1005
|
+
"""Stats du cache d'outils (declaree avant /api/tools/{server})."""
|
|
1006
|
+
from fastapi.responses import JSONResponse
|
|
1007
|
+
try:
|
|
1008
|
+
import tool_proxy as tp
|
|
1009
|
+
return tp.stats()
|
|
1010
|
+
except Exception as exc: # noqa: BLE001
|
|
1011
|
+
return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
@app.get("/api/tools/{server}")
|
|
1015
|
+
async def api_list_server_tools(server: str):
|
|
1016
|
+
"""Outils exposes par un serveur MCP cible."""
|
|
1017
|
+
try:
|
|
1018
|
+
import tool_proxy as tp
|
|
1019
|
+
except Exception as exc:
|
|
1020
|
+
return JSONResponse({"ok": False, "error": f"tool_proxy indisponible: {exc}"}, status_code=503)
|
|
1021
|
+
return await tp.list_server_tools(server)
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
@app.get("/api/live")
|
|
1025
|
+
async def api_live(request: Request, bucket: str = "hour", hours: int = 24):
|
|
1026
|
+
"""Flux SSE temps reel : pousse {metrics, series} a chaque nouvel evenement (<1 s)."""
|
|
1027
|
+
import sqlite3
|
|
1028
|
+
|
|
1029
|
+
def _sig():
|
|
1030
|
+
conn = sqlite3.connect(DB_FILE)
|
|
1031
|
+
try:
|
|
1032
|
+
return conn.execute("SELECT COALESCE(MAX(id),0), COUNT(*) FROM events").fetchone()
|
|
1033
|
+
finally:
|
|
1034
|
+
conn.close()
|
|
1035
|
+
|
|
1036
|
+
async def gen():
|
|
1037
|
+
last_sig, last_ping = None, time.time()
|
|
1038
|
+
while True:
|
|
1039
|
+
if await request.is_disconnected():
|
|
1040
|
+
return
|
|
1041
|
+
try:
|
|
1042
|
+
sig = _sig()
|
|
1043
|
+
now = time.time()
|
|
1044
|
+
if sig != last_sig:
|
|
1045
|
+
last_sig, last_ping = sig, now
|
|
1046
|
+
payload = {"metrics": await api_metrics(),
|
|
1047
|
+
"series": await api_metrics_series(bucket, hours)}
|
|
1048
|
+
yield f"data: {json.dumps(payload)}\n\n"
|
|
1049
|
+
elif now - last_ping > 12:
|
|
1050
|
+
last_ping = now
|
|
1051
|
+
yield ": keepalive\n\n"
|
|
1052
|
+
except Exception as exc: # noqa: BLE001
|
|
1053
|
+
_log.exception("api/live")
|
|
1054
|
+
yield f"event: error\ndata: {json.dumps({'error': str(exc)})}\n\n"
|
|
1055
|
+
await asyncio.sleep(2)
|
|
1056
|
+
await asyncio.sleep(0.7)
|
|
1057
|
+
|
|
1058
|
+
return StreamingResponse(gen(), media_type="text/event-stream",
|
|
1059
|
+
headers={"Cache-Control": "no-store", "X-Accel-Buffering": "no"})
|
|
1060
|
+
|
|
1061
|
+
|
|
1062
|
+
@app.get("/favicon.ico", include_in_schema=False)
|
|
1063
|
+
async def favicon_ico():
|
|
1064
|
+
from fastapi.responses import Response
|
|
1065
|
+
return Response(status_code=204)
|
|
1066
|
+
|
|
1067
|
+
|
|
1068
|
+
@app.get("/api/agent-events")
|
|
1069
|
+
async def api_agent_events(limit: int = 50, kind: str = ""):
|
|
1070
|
+
kinds = [k for k in (kind or "").split(",") if k] or None
|
|
1071
|
+
return {"events": icy_core.agent_events_recent(limit=limit, kinds=kinds)}
|
|
1072
|
+
|
|
1073
|
+
|
|
1074
|
+
@app.get("/dashboard", response_class=HTMLResponse)
|
|
1075
|
+
async def dashboard():
|
|
1076
|
+
"""Sert le dashboard existant (lecture seule, aucune modification)."""
|
|
1077
|
+
html = PROJECT_ROOT / "static" / "dashboard.html"
|
|
1078
|
+
if not html.exists():
|
|
1079
|
+
raise HTTPException(status_code=404, detail="dashboard.html absent")
|
|
1080
|
+
return HTMLResponse(html.read_text(encoding="utf-8"), headers={"Cache-Control": "no-store"})
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
# ==== Anti-cache API =========================================================
|
|
1084
|
+
# Les endpoints d'observabilite renvoyaient un corps JSON sans Cache-Control.
|
|
1085
|
+
# Chrome pouvait donc resservir un payload perime -> dashboard fige a 0 alors
|
|
1086
|
+
# que la base contenait des donnees. On force no-store sur toute l'API.
|
|
1087
|
+
@app.middleware("http")
|
|
1088
|
+
async def _no_store_api(request: Request, call_next):
|
|
1089
|
+
resp = await call_next(request)
|
|
1090
|
+
if request.url.path.startswith(("/api/", "/health")):
|
|
1091
|
+
resp.headers["Cache-Control"] = "no-store, must-revalidate"
|
|
1092
|
+
resp.headers["Pragma"] = "no-cache"
|
|
1093
|
+
resp.headers["Expires"] = "0"
|
|
1094
|
+
return resp
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
# ==== Assets du dashboard ====================================================
|
|
1098
|
+
# dashboard.html reference three.min.js / icywebgl.js / logo.svg a la RACINE.
|
|
1099
|
+
# Liste blanche stricte (pas de directory traversal) + montage /static.
|
|
1100
|
+
_ROOT_ASSETS = {"three.min.js": "application/javascript",
|
|
1101
|
+
"icywebgl.js": "application/javascript",
|
|
1102
|
+
"logo.svg": "image/svg+xml"}
|
|
1103
|
+
|
|
1104
|
+
|
|
1105
|
+
@app.get("/v1/models")
|
|
1106
|
+
async def models_endpoint(request: Request):
|
|
1107
|
+
"""Liste des modeles de l'upstream (meme traitement que /<upstream>/v1/models)."""
|
|
1108
|
+
return await handle_llm(request, "/v1/models")
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
@app.get("/{asset_name}")
|
|
1112
|
+
async def root_asset(asset_name: str):
|
|
1113
|
+
media = _ROOT_ASSETS.get(asset_name)
|
|
1114
|
+
path = PROJECT_ROOT / "static" / asset_name
|
|
1115
|
+
if media is None or not path.is_file():
|
|
1116
|
+
raise HTTPException(status_code=404, detail="not found")
|
|
1117
|
+
return FileResponse(path, media_type=media, headers={"Cache-Control": "no-cache"})
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
from fastapi.staticfiles import StaticFiles # noqa: E402
|
|
1121
|
+
app.mount("/static", StaticFiles(directory=str(PROJECT_ROOT / "static")), name="static")
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
# =============================================================================
|
|
1125
|
+
if __name__ == "__main__":
|
|
1126
|
+
import argparse
|
|
1127
|
+
|
|
1128
|
+
parser = argparse.ArgumentParser(description="IcyCache Gateway (multi-upstream)")
|
|
1129
|
+
parser.add_argument("--port", type=int, default=int(GW.get("port", PROXY_CFG.get("port", 8080))))
|
|
1130
|
+
parser.add_argument("--host", default=str(GW.get("host", PROXY_CFG.get("host", "127.0.0.1"))))
|
|
1131
|
+
parser.add_argument("--db", default=str(DB_FILE))
|
|
1132
|
+
args = parser.parse_args()
|
|
1133
|
+
|
|
1134
|
+
logging_level = CONFIG.get("logging", {}).get("level", "INFO")
|
|
1135
|
+
import logging
|
|
1136
|
+
logging.basicConfig(level=getattr(logging, logging_level, logging.INFO),
|
|
1137
|
+
format="%(asctime)s - IcyCache.gateway - %(levelname)s - %(message)s")
|
|
1138
|
+
print(f"IcyCache gateway -> http://{args.host}:{args.port}")
|
|
1139
|
+
print(f" upstreams : {', '.join(_upstream_names())}")
|
|
1140
|
+
print(f" default : {DEFAULT_UPSTREAM or '(aucun - explicite obligatoire)'}")
|
|
1141
|
+
print(f" websocket : ws://{args.host}:{args.port}/ws/events")
|
|
1142
|
+
uvicorn.run(app, host=args.host, port=args.port, log_level="warning")
|