superlocalmemory 3.5.7 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ATTRIBUTION.md +24 -0
- package/CHANGELOG.md +40 -0
- package/README.md +142 -35
- package/package.json +1 -1
- package/pyproject.toml +2 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/cache_cmd.py +198 -0
- package/src/superlocalmemory/cli/commands.py +100 -2
- package/src/superlocalmemory/cli/compress_cmd.py +179 -0
- package/src/superlocalmemory/cli/help_cmd.py +197 -0
- package/src/superlocalmemory/cli/main.py +122 -0
- package/src/superlocalmemory/cli/optimize_cmd.py +175 -0
- package/src/superlocalmemory/cli/optimize_constants.py +31 -0
- package/src/superlocalmemory/cli/proxy_cmd.py +95 -0
- package/src/superlocalmemory/core/config.py +5 -0
- package/src/superlocalmemory/core/engine.py +23 -0
- package/src/superlocalmemory/core/mcp_embedder_proxy.py +89 -0
- package/src/superlocalmemory/infra/process_reaper.py +12 -1
- package/src/superlocalmemory/llm/backbone.py +10 -4
- package/src/superlocalmemory/mcp/server.py +34 -0
- package/src/superlocalmemory/mcp/tools_v3.py +6 -2
- package/src/superlocalmemory/optimize/NOTICE +11 -0
- package/src/superlocalmemory/optimize/__init__.py +0 -0
- package/src/superlocalmemory/optimize/adapters/__init__.py +68 -0
- package/src/superlocalmemory/optimize/adapters/_agent_registry.py +120 -0
- package/src/superlocalmemory/optimize/adapters/anthropic_adapter.py +115 -0
- package/src/superlocalmemory/optimize/adapters/openai_adapter.py +125 -0
- package/src/superlocalmemory/optimize/adapters/wrap.py +188 -0
- package/src/superlocalmemory/optimize/cache/__init__.py +31 -0
- package/src/superlocalmemory/optimize/cache/boundary_store.py +455 -0
- package/src/superlocalmemory/optimize/cache/centroid_store.py +158 -0
- package/src/superlocalmemory/optimize/cache/context_key.py +67 -0
- package/src/superlocalmemory/optimize/cache/exact.py +85 -0
- package/src/superlocalmemory/optimize/cache/invalidation.py +36 -0
- package/src/superlocalmemory/optimize/cache/key_builder.py +98 -0
- package/src/superlocalmemory/optimize/cache/manager.py +452 -0
- package/src/superlocalmemory/optimize/cache/semantic.py +568 -0
- package/src/superlocalmemory/optimize/cache/stampede.py +50 -0
- package/src/superlocalmemory/optimize/compress/__init__.py +17 -0
- package/src/superlocalmemory/optimize/compress/align.py +153 -0
- package/src/superlocalmemory/optimize/compress/ccr.py +157 -0
- package/src/superlocalmemory/optimize/compress/extractive_code.py +311 -0
- package/src/superlocalmemory/optimize/compress/extractive_json.py +72 -0
- package/src/superlocalmemory/optimize/compress/prose_llmlingua.py +77 -0
- package/src/superlocalmemory/optimize/compress/router.py +548 -0
- package/src/superlocalmemory/optimize/config/__init__.py +35 -0
- package/src/superlocalmemory/optimize/config/defaults.py +48 -0
- package/src/superlocalmemory/optimize/config/schema.py +255 -0
- package/src/superlocalmemory/optimize/config/store.py +209 -0
- package/src/superlocalmemory/optimize/metrics/__init__.py +8 -0
- package/src/superlocalmemory/optimize/metrics/counters.py +138 -0
- package/src/superlocalmemory/optimize/metrics/estimator.py +90 -0
- package/src/superlocalmemory/optimize/metrics/exporters.py +77 -0
- package/src/superlocalmemory/optimize/metrics/persistence.py +115 -0
- package/src/superlocalmemory/optimize/proxy/__init__.py +28 -0
- package/src/superlocalmemory/optimize/proxy/_helpers.py +257 -0
- package/src/superlocalmemory/optimize/proxy/anthropic_surface.py +171 -0
- package/src/superlocalmemory/optimize/proxy/gemini_surface.py +121 -0
- package/src/superlocalmemory/optimize/proxy/lifecycle.py +126 -0
- package/src/superlocalmemory/optimize/proxy/openai_surface.py +125 -0
- package/src/superlocalmemory/optimize/proxy/server.py +151 -0
- package/src/superlocalmemory/optimize/storage/__init__.py +0 -0
- package/src/superlocalmemory/optimize/storage/db.py +1016 -0
- package/src/superlocalmemory/optimize/storage/schema.py +184 -0
- package/src/superlocalmemory/server/routes/optimize.py +166 -0
- package/src/superlocalmemory/server/routes/v3_api.py +63 -1
- package/src/superlocalmemory/server/unified_daemon.py +105 -0
- package/src/superlocalmemory/ui/index.html +98 -0
- package/src/superlocalmemory/ui/js/ng-shell.js +2 -1
- package/src/superlocalmemory/ui/js/optimize.js +173 -0
- package/src/superlocalmemory.egg-info/PKG-INFO +2 -1
- package/src/superlocalmemory.egg-info/SOURCES.txt +51 -0
- package/src/superlocalmemory.egg-info/requires.txt +1 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""anthropic_surface.py — Anthropic Messages + count_tokens + models surfaces."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
|
|
8
|
+
from fastapi.requests import Request
|
|
9
|
+
from fastapi.responses import Response
|
|
10
|
+
|
|
11
|
+
from superlocalmemory.optimize.proxy._helpers import (
|
|
12
|
+
_ANTHROPIC_FORWARD_HEADERS,
|
|
13
|
+
_MAX_REQUEST_BODY_BYTES,
|
|
14
|
+
_body_has_tools,
|
|
15
|
+
_build_forward_headers,
|
|
16
|
+
_fail_open_forward,
|
|
17
|
+
_filter_response_headers,
|
|
18
|
+
_redact_headers,
|
|
19
|
+
_safe_cache_check,
|
|
20
|
+
_safe_cache_hit_callbacks,
|
|
21
|
+
_safe_cache_store,
|
|
22
|
+
_stream_forward,
|
|
23
|
+
)
|
|
24
|
+
from superlocalmemory.optimize.proxy.lifecycle import ProviderResponse, ProxyRequest
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger("slm.optimize.proxy.anthropic")
|
|
27
|
+
|
|
28
|
+
_UPSTREAM_BASE = "https://api.anthropic.com"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
async def handle_messages(proxy: object, request: Request) -> Response:
|
|
32
|
+
request_id = await proxy.next_request_id()
|
|
33
|
+
upstream_url = f"{_UPSTREAM_BASE}/v1/messages"
|
|
34
|
+
|
|
35
|
+
cl_header = request.headers.get("content-length")
|
|
36
|
+
if cl_header is not None:
|
|
37
|
+
try:
|
|
38
|
+
if int(cl_header) > _MAX_REQUEST_BODY_BYTES:
|
|
39
|
+
return Response(
|
|
40
|
+
content=b'{"type":"error","error":{"type":"invalid_request_error",'
|
|
41
|
+
b'"message":"Request body too large (max 10 MB)"}}',
|
|
42
|
+
status_code=413,
|
|
43
|
+
media_type="application/json",
|
|
44
|
+
)
|
|
45
|
+
except ValueError:
|
|
46
|
+
pass
|
|
47
|
+
|
|
48
|
+
try:
|
|
49
|
+
body_bytes = await request.body()
|
|
50
|
+
if len(body_bytes) > _MAX_REQUEST_BODY_BYTES:
|
|
51
|
+
return Response(
|
|
52
|
+
content=b'{"type":"error","error":{"type":"invalid_request_error",'
|
|
53
|
+
b'"message":"Request body too large (max 10 MB)"}}',
|
|
54
|
+
status_code=413,
|
|
55
|
+
media_type="application/json",
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
body = json.loads(body_bytes)
|
|
60
|
+
except json.JSONDecodeError as exc:
|
|
61
|
+
logger.warning("[%s] body parse failed, raw forward: %s", request_id, exc)
|
|
62
|
+
return await _fail_open_forward(proxy, request, upstream_url)
|
|
63
|
+
|
|
64
|
+
stream = bool(body.get("stream", False))
|
|
65
|
+
has_tools = _body_has_tools(body)
|
|
66
|
+
ctx = ProxyRequest(
|
|
67
|
+
provider="anthropic",
|
|
68
|
+
method="POST",
|
|
69
|
+
path="/v1/messages",
|
|
70
|
+
headers=_redact_headers(dict(request.headers)),
|
|
71
|
+
body=body,
|
|
72
|
+
body_bytes=body_bytes,
|
|
73
|
+
request_id=request_id,
|
|
74
|
+
stream=stream,
|
|
75
|
+
has_tools=has_tools,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
if stream:
|
|
79
|
+
fwd_headers = _build_forward_headers(request, _ANTHROPIC_FORWARD_HEADERS)
|
|
80
|
+
fwd_headers["content-length"] = str(len(body_bytes))
|
|
81
|
+
return await _stream_forward(
|
|
82
|
+
proxy, request_id, fwd_headers, body_bytes, upstream_url
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
cache_result = None
|
|
86
|
+
if not has_tools and proxy.hooks.cache:
|
|
87
|
+
cache_result = await _safe_cache_check(proxy.hooks, ctx)
|
|
88
|
+
if cache_result.hit and cache_result.data:
|
|
89
|
+
logger.debug("[%s] cache HIT key=%s", request_id, cache_result.cache_key)
|
|
90
|
+
await _safe_cache_hit_callbacks(
|
|
91
|
+
proxy.hooks, ctx, cache_result.data, tokens_saved=0
|
|
92
|
+
)
|
|
93
|
+
return Response(
|
|
94
|
+
content=cache_result.data,
|
|
95
|
+
status_code=200,
|
|
96
|
+
media_type="application/json",
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
outbound_bytes = body_bytes
|
|
100
|
+
if proxy.hooks.compress:
|
|
101
|
+
compress_result = await _safe_compress(proxy.hooks, ctx)
|
|
102
|
+
if compress_result.body_bytes != body_bytes:
|
|
103
|
+
outbound_bytes = compress_result.body_bytes
|
|
104
|
+
|
|
105
|
+
fwd_headers = _build_forward_headers(request, _ANTHROPIC_FORWARD_HEADERS)
|
|
106
|
+
fwd_headers["content-length"] = str(len(outbound_bytes))
|
|
107
|
+
|
|
108
|
+
upstream_resp = await proxy.http_client.post(
|
|
109
|
+
upstream_url, content=outbound_bytes, headers=fwd_headers,
|
|
110
|
+
)
|
|
111
|
+
resp_bytes = upstream_resp.content
|
|
112
|
+
|
|
113
|
+
if (
|
|
114
|
+
upstream_resp.status_code == 200
|
|
115
|
+
and not has_tools
|
|
116
|
+
and proxy.hooks.cache
|
|
117
|
+
and cache_result is not None
|
|
118
|
+
and cache_result.cache_key
|
|
119
|
+
):
|
|
120
|
+
_prov_resp = ProviderResponse(
|
|
121
|
+
modified=False, body={}, body_bytes=resp_bytes,
|
|
122
|
+
tokens_before=0, tokens_after=0, strategy="none",
|
|
123
|
+
)
|
|
124
|
+
await _safe_cache_store(proxy.hooks, ctx, _prov_resp)
|
|
125
|
+
|
|
126
|
+
return Response(
|
|
127
|
+
content=resp_bytes,
|
|
128
|
+
status_code=upstream_resp.status_code,
|
|
129
|
+
media_type="application/json",
|
|
130
|
+
headers=_filter_response_headers(dict(upstream_resp.headers)),
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
except Exception as exc:
|
|
134
|
+
logger.error("[%s] handle_messages unhandled exc=%r — fail-open", request_id, exc)
|
|
135
|
+
return await _fail_open_forward(proxy, request, upstream_url)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
async def handle_count_tokens(proxy: object, request: Request) -> Response:
|
|
139
|
+
request_id = await proxy.next_request_id()
|
|
140
|
+
upstream_url = f"{_UPSTREAM_BASE}/v1/messages/count_tokens"
|
|
141
|
+
try:
|
|
142
|
+
body_bytes = await request.body()
|
|
143
|
+
fwd_headers = _build_forward_headers(request, _ANTHROPIC_FORWARD_HEADERS)
|
|
144
|
+
fwd_headers["content-length"] = str(len(body_bytes))
|
|
145
|
+
upstream_resp = await proxy.http_client.post(
|
|
146
|
+
upstream_url, content=body_bytes, headers=fwd_headers,
|
|
147
|
+
)
|
|
148
|
+
return Response(
|
|
149
|
+
content=upstream_resp.content,
|
|
150
|
+
status_code=upstream_resp.status_code,
|
|
151
|
+
media_type="application/json",
|
|
152
|
+
)
|
|
153
|
+
except Exception as exc:
|
|
154
|
+
logger.error("[%s] handle_count_tokens exc=%r — fail-open", request_id, exc)
|
|
155
|
+
return await _fail_open_forward(proxy, request, upstream_url)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
async def handle_models(proxy: object, request: Request) -> Response:
|
|
159
|
+
request_id = await proxy.next_request_id()
|
|
160
|
+
upstream_url = f"{_UPSTREAM_BASE}/v1/models"
|
|
161
|
+
try:
|
|
162
|
+
fwd_headers = _build_forward_headers(request, _ANTHROPIC_FORWARD_HEADERS)
|
|
163
|
+
upstream_resp = await proxy.http_client.get(upstream_url, headers=fwd_headers)
|
|
164
|
+
return Response(
|
|
165
|
+
content=upstream_resp.content,
|
|
166
|
+
status_code=upstream_resp.status_code,
|
|
167
|
+
media_type="application/json",
|
|
168
|
+
)
|
|
169
|
+
except Exception as exc:
|
|
170
|
+
logger.error("[%s] handle_models exc=%r — fail-open", request_id, exc)
|
|
171
|
+
return await _fail_open_forward(proxy, request, upstream_url)
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""gemini_surface.py — Gemini native and OpenAI-compat surfaces."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import re
|
|
7
|
+
import urllib.parse
|
|
8
|
+
|
|
9
|
+
from fastapi.requests import Request
|
|
10
|
+
from fastapi.responses import Response
|
|
11
|
+
|
|
12
|
+
from superlocalmemory.optimize.proxy._helpers import (
|
|
13
|
+
_GEMINI_NATIVE_FORWARD_HEADERS,
|
|
14
|
+
_GEMINI_OPENAI_COMPAT_FORWARD_HEADERS,
|
|
15
|
+
_fail_open_forward,
|
|
16
|
+
_filter_response_headers,
|
|
17
|
+
_stream_forward,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger("slm.optimize.proxy.gemini")
|
|
21
|
+
|
|
22
|
+
_GEMINI_UPSTREAM_BASE = "https://generativelanguage.googleapis.com"
|
|
23
|
+
|
|
24
|
+
# SSRF guard: accept BOTH `models/<name>:<method>` AND `<name>:<method>`
|
|
25
|
+
# (FastAPI's `:path` converter may strip the `models/` segment depending on
|
|
26
|
+
# route declaration; we normalize on the server side either way).
|
|
27
|
+
_GEMINI_PATH_RE = re.compile(
|
|
28
|
+
r"^(?:models/)?[a-zA-Z0-9._\-]{1,128}:(generateContent|streamGenerateContent|countTokens)$"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
_GEMINI_ALLOWED_QUERY_PARAMS = frozenset(["pagesize", "pagetoken"])
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _validate_gemini_path(model_and_method: str) -> bool:
|
|
35
|
+
return bool(_GEMINI_PATH_RE.match(model_and_method))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
async def handle_gemini_native(
|
|
39
|
+
proxy: object,
|
|
40
|
+
request: Request,
|
|
41
|
+
model_and_method: str,
|
|
42
|
+
) -> Response:
|
|
43
|
+
request_id = await proxy.next_request_id()
|
|
44
|
+
|
|
45
|
+
if not _validate_gemini_path(model_and_method):
|
|
46
|
+
logger.warning(
|
|
47
|
+
"[%s] handle_gemini_native: rejected invalid path param=%r (SSRF guard)",
|
|
48
|
+
request_id, model_and_method,
|
|
49
|
+
)
|
|
50
|
+
return Response(
|
|
51
|
+
content=b'{"error":{"code":400,"message":"Invalid model/method path",'
|
|
52
|
+
b'"status":"INVALID_ARGUMENT"}}',
|
|
53
|
+
status_code=400,
|
|
54
|
+
media_type="application/json",
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
upstream_url = f"{_GEMINI_UPSTREAM_BASE}/v1beta/{model_and_method}"
|
|
58
|
+
|
|
59
|
+
try:
|
|
60
|
+
body_bytes = await request.body()
|
|
61
|
+
fwd_headers = {
|
|
62
|
+
k: v for k, v in request.headers.items()
|
|
63
|
+
if k.lower() in _GEMINI_NATIVE_FORWARD_HEADERS
|
|
64
|
+
}
|
|
65
|
+
fwd_headers["content-length"] = str(len(body_bytes))
|
|
66
|
+
|
|
67
|
+
stream = "streamGenerateContent" in model_and_method
|
|
68
|
+
if stream:
|
|
69
|
+
allowed = {
|
|
70
|
+
k: v for k, v in request.query_params.items()
|
|
71
|
+
if k.lower() in _GEMINI_ALLOWED_QUERY_PARAMS
|
|
72
|
+
}
|
|
73
|
+
allowed["alt"] = "sse"
|
|
74
|
+
upstream_url = f"{upstream_url}?{urllib.parse.urlencode(allowed)}"
|
|
75
|
+
return await _stream_forward(
|
|
76
|
+
proxy, request_id, fwd_headers, body_bytes, upstream_url
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
upstream_resp = await proxy.http_client.post(
|
|
80
|
+
upstream_url, content=body_bytes, headers=fwd_headers,
|
|
81
|
+
)
|
|
82
|
+
return Response(
|
|
83
|
+
content=upstream_resp.content,
|
|
84
|
+
status_code=upstream_resp.status_code,
|
|
85
|
+
media_type="application/json",
|
|
86
|
+
headers=_filter_response_headers(dict(upstream_resp.headers)),
|
|
87
|
+
)
|
|
88
|
+
except Exception as exc:
|
|
89
|
+
logger.error("[%s] handle_gemini_native exc=%r — fail-open", request_id, exc)
|
|
90
|
+
return await _fail_open_forward(proxy, request, upstream_url)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
async def handle_gemini_openai_compat(proxy: object, request: Request) -> Response:
|
|
94
|
+
local_path = request.url.path
|
|
95
|
+
upstream_url = f"{_GEMINI_UPSTREAM_BASE}{local_path}"
|
|
96
|
+
request_id = await proxy.next_request_id()
|
|
97
|
+
try:
|
|
98
|
+
body_bytes = await request.body()
|
|
99
|
+
fwd_headers = {
|
|
100
|
+
k: v for k, v in request.headers.items()
|
|
101
|
+
if k.lower() in _GEMINI_OPENAI_COMPAT_FORWARD_HEADERS
|
|
102
|
+
}
|
|
103
|
+
if body_bytes:
|
|
104
|
+
fwd_headers["content-length"] = str(len(body_bytes))
|
|
105
|
+
upstream_resp = await proxy.http_client.request(
|
|
106
|
+
method=request.method,
|
|
107
|
+
url=upstream_url,
|
|
108
|
+
content=body_bytes if body_bytes else None,
|
|
109
|
+
headers=fwd_headers,
|
|
110
|
+
)
|
|
111
|
+
return Response(
|
|
112
|
+
content=upstream_resp.content,
|
|
113
|
+
status_code=upstream_resp.status_code,
|
|
114
|
+
media_type="application/json",
|
|
115
|
+
headers=_filter_response_headers(dict(upstream_resp.headers)),
|
|
116
|
+
)
|
|
117
|
+
except Exception as exc:
|
|
118
|
+
logger.error(
|
|
119
|
+
"[%s] handle_gemini_openai_compat exc=%r — fail-open", request_id, exc
|
|
120
|
+
)
|
|
121
|
+
return await _fail_open_forward(proxy, request, upstream_url)
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""lifecycle.py — Hook Protocol interfaces per INTERFACE-CONTRACT §3 (FROZEN).
|
|
2
|
+
|
|
3
|
+
TYPE NAME CANONICAL RULE:
|
|
4
|
+
ProxyRequest — request DTO
|
|
5
|
+
CachedResponse — cache hit/miss result
|
|
6
|
+
ProviderResponse — compress result / passthrough body
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import dataclasses
|
|
12
|
+
from typing import Any, Protocol, runtime_checkable
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# ─── Data types ──────────────────────────────────────────────────────────────
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclasses.dataclass(frozen=True)
|
|
19
|
+
class ProxyRequest:
|
|
20
|
+
"""Immutable request snapshot passed to every hook.
|
|
21
|
+
|
|
22
|
+
headers MUST be redacted before construction (CWE-532 guard).
|
|
23
|
+
"""
|
|
24
|
+
provider: str
|
|
25
|
+
method: str
|
|
26
|
+
path: str
|
|
27
|
+
headers: dict
|
|
28
|
+
body: dict
|
|
29
|
+
body_bytes: bytes
|
|
30
|
+
request_id: str
|
|
31
|
+
stream: bool
|
|
32
|
+
has_tools: bool
|
|
33
|
+
|
|
34
|
+
def __repr__(self) -> str:
|
|
35
|
+
return (
|
|
36
|
+
f"ProxyRequest(provider={self.provider!r}, method={self.method!r}, "
|
|
37
|
+
f"path={self.path!r}, request_id={self.request_id!r}, "
|
|
38
|
+
f"stream={self.stream}, has_tools={self.has_tools})"
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclasses.dataclass
|
|
43
|
+
class CachedResponse:
|
|
44
|
+
"""Cache check result."""
|
|
45
|
+
hit: bool
|
|
46
|
+
data: bytes | None
|
|
47
|
+
cache_key: str
|
|
48
|
+
ttl_seconds: int
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclasses.dataclass
|
|
52
|
+
class ProviderResponse:
|
|
53
|
+
"""Compress-modified body container / passthrough body shape."""
|
|
54
|
+
modified: bool
|
|
55
|
+
body: dict
|
|
56
|
+
body_bytes: bytes
|
|
57
|
+
tokens_before: int
|
|
58
|
+
tokens_after: int
|
|
59
|
+
strategy: str
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ─── Hook Protocols — INTERFACE-CONTRACT §3 ────────────────────────────────
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@runtime_checkable
|
|
66
|
+
class CacheHook(Protocol):
|
|
67
|
+
def check(self, req: ProxyRequest) -> CachedResponse | None: ...
|
|
68
|
+
def store(self, req: ProxyRequest, resp: ProviderResponse) -> None: ...
|
|
69
|
+
def on_hit(self, req: ProxyRequest, resp: bytes, tokens_saved: int) -> None: ...
|
|
70
|
+
def on_miss(self, req: ProxyRequest) -> None: ...
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@runtime_checkable
|
|
74
|
+
class CompressHook(Protocol):
|
|
75
|
+
def compress(self, req: ProxyRequest) -> ProxyRequest: ...
|
|
76
|
+
def on_compress(self, before_tokens: int, after_tokens: int, lossy: bool) -> None: ...
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclasses.dataclass
|
|
80
|
+
class HookChain:
|
|
81
|
+
cache: CacheHook | None = None
|
|
82
|
+
compress: CompressHook | None = None
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def empty(cls) -> "HookChain":
|
|
86
|
+
return cls(cache=None, compress=None)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# ─── Lifecycle exports (INTERFACE-CONTRACT v2.2) ───────────────────────────
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def ensure_proxy_running() -> bool:
|
|
93
|
+
"""Ensure the proxy is configured AND alive (liveness probe).
|
|
94
|
+
|
|
95
|
+
Checks two conditions in order:
|
|
96
|
+
1. ``proxy_enabled`` is True in optimize.json (config gate).
|
|
97
|
+
2. The proxy HTTP server responds to GET /health within 1 second (liveness
|
|
98
|
+
gate). This verifies that ProxyApp.startup() was actually called and
|
|
99
|
+
the httpx.AsyncClient is non-None — a detail the config flag alone
|
|
100
|
+
cannot confirm.
|
|
101
|
+
|
|
102
|
+
Returns True only when both gates pass, False otherwise.
|
|
103
|
+
"""
|
|
104
|
+
try:
|
|
105
|
+
from superlocalmemory.optimize.config import get_optimize_config
|
|
106
|
+
cfg = get_optimize_config()
|
|
107
|
+
if not cfg.proxy_enabled:
|
|
108
|
+
return False
|
|
109
|
+
except Exception:
|
|
110
|
+
return False
|
|
111
|
+
|
|
112
|
+
# Liveness probe — confirm the daemon is actually listening.
|
|
113
|
+
try:
|
|
114
|
+
import urllib.request
|
|
115
|
+
port = proxy_port()
|
|
116
|
+
url = f"http://127.0.0.1:{port}/health"
|
|
117
|
+
req = urllib.request.Request(url, method="GET")
|
|
118
|
+
with urllib.request.urlopen(req, timeout=1) as resp:
|
|
119
|
+
return resp.status == 200
|
|
120
|
+
except Exception:
|
|
121
|
+
return False
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def proxy_port() -> int:
|
|
125
|
+
"""Returns the proxy port (always 8765 — shared with SLM daemon)."""
|
|
126
|
+
return 8765
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""openai_surface.py — OpenAI /v1/chat/completions and /v1/embeddings."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
|
|
8
|
+
from fastapi.requests import Request
|
|
9
|
+
from fastapi.responses import Response
|
|
10
|
+
|
|
11
|
+
from superlocalmemory.optimize.proxy._helpers import (
|
|
12
|
+
_OPENAI_FORWARD_HEADERS,
|
|
13
|
+
_body_has_tools,
|
|
14
|
+
_build_forward_headers,
|
|
15
|
+
_fail_open_forward,
|
|
16
|
+
_filter_response_headers,
|
|
17
|
+
_redact_headers,
|
|
18
|
+
_safe_cache_check,
|
|
19
|
+
_safe_cache_hit_callbacks,
|
|
20
|
+
_safe_cache_store,
|
|
21
|
+
_stream_forward,
|
|
22
|
+
)
|
|
23
|
+
from superlocalmemory.optimize.proxy.lifecycle import ProviderResponse, ProxyRequest
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger("slm.optimize.proxy.openai")
|
|
26
|
+
|
|
27
|
+
_UPSTREAM_BASE = "https://api.openai.com"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
async def handle_chat_completions(proxy: object, request: Request) -> Response:
|
|
31
|
+
request_id = await proxy.next_request_id()
|
|
32
|
+
upstream_url = f"{_UPSTREAM_BASE}/v1/chat/completions"
|
|
33
|
+
try:
|
|
34
|
+
body_bytes = await request.body()
|
|
35
|
+
try:
|
|
36
|
+
body = json.loads(body_bytes)
|
|
37
|
+
except json.JSONDecodeError:
|
|
38
|
+
return await _fail_open_forward(proxy, request, upstream_url)
|
|
39
|
+
|
|
40
|
+
stream = bool(body.get("stream", False))
|
|
41
|
+
has_tools = _body_has_tools(body)
|
|
42
|
+
ctx = ProxyRequest(
|
|
43
|
+
provider="openai", method="POST", path="/v1/chat/completions",
|
|
44
|
+
headers=_redact_headers(dict(request.headers)),
|
|
45
|
+
body=body, body_bytes=body_bytes,
|
|
46
|
+
request_id=request_id, stream=stream, has_tools=has_tools,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
if stream:
|
|
50
|
+
fwd_headers = _build_forward_headers(request, _OPENAI_FORWARD_HEADERS)
|
|
51
|
+
fwd_headers["content-length"] = str(len(body_bytes))
|
|
52
|
+
return await _stream_forward(
|
|
53
|
+
proxy, request_id, fwd_headers, body_bytes, upstream_url
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
cache_result = None
|
|
57
|
+
if not has_tools and proxy.hooks.cache:
|
|
58
|
+
cache_result = await _safe_cache_check(proxy.hooks, ctx)
|
|
59
|
+
if cache_result.hit and cache_result.data:
|
|
60
|
+
await _safe_cache_hit_callbacks(
|
|
61
|
+
proxy.hooks, ctx, cache_result.data, 0
|
|
62
|
+
)
|
|
63
|
+
return Response(
|
|
64
|
+
content=cache_result.data,
|
|
65
|
+
status_code=200,
|
|
66
|
+
media_type="application/json",
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
outbound_bytes = body_bytes
|
|
70
|
+
if proxy.hooks.compress:
|
|
71
|
+
compress_result = await _safe_compress(proxy.hooks, ctx)
|
|
72
|
+
if compress_result.body_bytes != body_bytes:
|
|
73
|
+
outbound_bytes = compress_result.body_bytes
|
|
74
|
+
|
|
75
|
+
fwd_headers = _build_forward_headers(request, _OPENAI_FORWARD_HEADERS)
|
|
76
|
+
fwd_headers["content-length"] = str(len(outbound_bytes))
|
|
77
|
+
|
|
78
|
+
upstream_resp = await proxy.http_client.post(
|
|
79
|
+
upstream_url, content=outbound_bytes, headers=fwd_headers,
|
|
80
|
+
)
|
|
81
|
+
resp_bytes = upstream_resp.content
|
|
82
|
+
|
|
83
|
+
if (
|
|
84
|
+
upstream_resp.status_code == 200
|
|
85
|
+
and not has_tools
|
|
86
|
+
and proxy.hooks.cache
|
|
87
|
+
and cache_result is not None
|
|
88
|
+
and cache_result.cache_key
|
|
89
|
+
):
|
|
90
|
+
_prov_resp = ProviderResponse(
|
|
91
|
+
modified=False, body={}, body_bytes=resp_bytes,
|
|
92
|
+
tokens_before=0, tokens_after=0, strategy="none",
|
|
93
|
+
)
|
|
94
|
+
await _safe_cache_store(proxy.hooks, ctx, _prov_resp)
|
|
95
|
+
|
|
96
|
+
return Response(
|
|
97
|
+
content=resp_bytes,
|
|
98
|
+
status_code=upstream_resp.status_code,
|
|
99
|
+
media_type="application/json",
|
|
100
|
+
headers=_filter_response_headers(dict(upstream_resp.headers)),
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
except Exception as exc:
|
|
104
|
+
logger.error("[%s] handle_chat_completions exc=%r — fail-open", request_id, exc)
|
|
105
|
+
return await _fail_open_forward(proxy, request, upstream_url)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
async def handle_embeddings(proxy: object, request: Request) -> Response:
|
|
109
|
+
request_id = await proxy.next_request_id()
|
|
110
|
+
upstream_url = f"{_UPSTREAM_BASE}/v1/embeddings"
|
|
111
|
+
try:
|
|
112
|
+
body_bytes = await request.body()
|
|
113
|
+
fwd_headers = _build_forward_headers(request, _OPENAI_FORWARD_HEADERS)
|
|
114
|
+
fwd_headers["content-length"] = str(len(body_bytes))
|
|
115
|
+
upstream_resp = await proxy.http_client.post(
|
|
116
|
+
upstream_url, content=body_bytes, headers=fwd_headers,
|
|
117
|
+
)
|
|
118
|
+
return Response(
|
|
119
|
+
content=upstream_resp.content,
|
|
120
|
+
status_code=upstream_resp.status_code,
|
|
121
|
+
media_type="application/json",
|
|
122
|
+
)
|
|
123
|
+
except Exception as exc:
|
|
124
|
+
logger.error("[%s] handle_embeddings exc=%r — fail-open", request_id, exc)
|
|
125
|
+
return await _fail_open_forward(proxy, request, upstream_url)
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""server.py — ProxyApp and build_proxy_router()."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
import time
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
from fastapi import APIRouter
|
|
12
|
+
from fastapi.requests import Request
|
|
13
|
+
from fastapi.responses import Response
|
|
14
|
+
|
|
15
|
+
from superlocalmemory.optimize.config.schema import OptimizeConfig
|
|
16
|
+
from superlocalmemory.optimize.proxy.lifecycle import HookChain
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger("slm.optimize.proxy")
|
|
19
|
+
|
|
20
|
+
_PROXY_VERSION = "3.6.0"
|
|
21
|
+
_REQUEST_TIMEOUT_S = 300.0
|
|
22
|
+
_CONNECT_TIMEOUT_S = 10.0
|
|
23
|
+
_MAX_CONNECTIONS = 100
|
|
24
|
+
_MAX_KEEPALIVE = 20
|
|
25
|
+
_MAX_REQUEST_BODY_BYTES = 10 * 1024 * 1024 # 10 MB
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ProxyApp:
|
|
29
|
+
"""Core proxy. Holds httpx client + hook chain.
|
|
30
|
+
|
|
31
|
+
Lifecycle:
|
|
32
|
+
create_app() → ProxyApp() → application.state.optimize_proxy = proxy
|
|
33
|
+
lifespan startup → await proxy.startup()
|
|
34
|
+
lifespan shutdown → await proxy.shutdown()
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def __init__(self, config: OptimizeConfig) -> None:
|
|
38
|
+
self.config = config
|
|
39
|
+
self.hooks: HookChain = HookChain.empty()
|
|
40
|
+
self.http_client: httpx.AsyncClient | None = None
|
|
41
|
+
self._request_counter: int = 0
|
|
42
|
+
self._counter_lock = asyncio.Lock()
|
|
43
|
+
|
|
44
|
+
async def startup(self) -> None:
|
|
45
|
+
self.http_client = httpx.AsyncClient(
|
|
46
|
+
timeout=httpx.Timeout(
|
|
47
|
+
connect=_CONNECT_TIMEOUT_S,
|
|
48
|
+
read=_REQUEST_TIMEOUT_S,
|
|
49
|
+
write=_REQUEST_TIMEOUT_S,
|
|
50
|
+
pool=_CONNECT_TIMEOUT_S,
|
|
51
|
+
),
|
|
52
|
+
limits=httpx.Limits(
|
|
53
|
+
max_connections=_MAX_CONNECTIONS,
|
|
54
|
+
max_keepalive_connections=_MAX_KEEPALIVE,
|
|
55
|
+
),
|
|
56
|
+
follow_redirects=False,
|
|
57
|
+
)
|
|
58
|
+
self.hooks = _load_hooks(self.config)
|
|
59
|
+
logger.info(
|
|
60
|
+
"slm.optimize.proxy started version=%s port=8765 "
|
|
61
|
+
"cache_hook=%s compress_hook=%s",
|
|
62
|
+
_PROXY_VERSION,
|
|
63
|
+
type(self.hooks.cache).__name__ if self.hooks.cache else "None",
|
|
64
|
+
type(self.hooks.compress).__name__ if self.hooks.compress else "None",
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
async def shutdown(self) -> None:
|
|
68
|
+
if self.http_client:
|
|
69
|
+
await self.http_client.aclose()
|
|
70
|
+
self.http_client = None
|
|
71
|
+
logger.info("slm.optimize.proxy shut down")
|
|
72
|
+
|
|
73
|
+
async def next_request_id(self) -> str:
|
|
74
|
+
async with self._counter_lock:
|
|
75
|
+
self._request_counter += 1
|
|
76
|
+
return f"slm_{int(time.monotonic() * 1000)}_{self._request_counter:06d}"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def build_proxy_router(proxy: ProxyApp) -> APIRouter:
|
|
80
|
+
"""Build and return the FastAPI router for all proxy surfaces."""
|
|
81
|
+
from superlocalmemory.optimize.proxy.anthropic_surface import (
|
|
82
|
+
handle_count_tokens,
|
|
83
|
+
handle_messages,
|
|
84
|
+
handle_models,
|
|
85
|
+
)
|
|
86
|
+
from superlocalmemory.optimize.proxy.gemini_surface import (
|
|
87
|
+
handle_gemini_native,
|
|
88
|
+
handle_gemini_openai_compat,
|
|
89
|
+
)
|
|
90
|
+
from superlocalmemory.optimize.proxy.openai_surface import (
|
|
91
|
+
handle_chat_completions,
|
|
92
|
+
handle_embeddings,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
router = APIRouter(tags=["slm-optimize-proxy"])
|
|
96
|
+
|
|
97
|
+
@router.post("/v1/messages")
|
|
98
|
+
async def messages_route(request: Request) -> Response:
|
|
99
|
+
return await handle_messages(proxy, request)
|
|
100
|
+
|
|
101
|
+
@router.post("/v1/messages/count_tokens")
|
|
102
|
+
async def count_tokens_route(request: Request) -> Response:
|
|
103
|
+
return await handle_count_tokens(proxy, request)
|
|
104
|
+
|
|
105
|
+
@router.get("/v1/models")
|
|
106
|
+
async def models_route(request: Request) -> Response:
|
|
107
|
+
return await handle_models(proxy, request)
|
|
108
|
+
|
|
109
|
+
@router.post("/v1/chat/completions")
|
|
110
|
+
async def chat_completions_route(request: Request) -> Response:
|
|
111
|
+
return await handle_chat_completions(proxy, request)
|
|
112
|
+
|
|
113
|
+
@router.post("/v1/embeddings")
|
|
114
|
+
async def embeddings_route(request: Request) -> Response:
|
|
115
|
+
return await handle_embeddings(proxy, request)
|
|
116
|
+
|
|
117
|
+
@router.post("/v1beta/models/{model_and_method:path}")
|
|
118
|
+
async def gemini_native_route(
|
|
119
|
+
request: Request, model_and_method: str
|
|
120
|
+
) -> Response:
|
|
121
|
+
return await handle_gemini_native(proxy, request, model_and_method)
|
|
122
|
+
|
|
123
|
+
@router.post("/v1beta/openai/chat/completions")
|
|
124
|
+
async def gemini_openai_post_route(request: Request) -> Response:
|
|
125
|
+
return await handle_gemini_openai_compat(proxy, request)
|
|
126
|
+
|
|
127
|
+
@router.get("/v1beta/openai/models")
|
|
128
|
+
async def gemini_openai_models_route(request: Request) -> Response:
|
|
129
|
+
return await handle_gemini_openai_compat(proxy, request)
|
|
130
|
+
|
|
131
|
+
return router
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _load_hooks(config: OptimizeConfig) -> HookChain:
|
|
135
|
+
cache_hook = None
|
|
136
|
+
compress_hook = None
|
|
137
|
+
|
|
138
|
+
if config.cache_enabled:
|
|
139
|
+
try:
|
|
140
|
+
from superlocalmemory.optimize.cache.manager import CacheManager
|
|
141
|
+
cache_hook = CacheManager.get_instance()
|
|
142
|
+
except Exception as exc:
|
|
143
|
+
logger.warning(
|
|
144
|
+
"cache hook load failed (proxy continues without cache): %s", exc
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
if config.compress_enabled:
|
|
148
|
+
# Compress is Phase 2; not wired in P1. P1 keeps the slot for the seam.
|
|
149
|
+
pass
|
|
150
|
+
|
|
151
|
+
return HookChain(cache=cache_hook, compress=compress_hook)
|
|
File without changes
|