smol-llm-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smol_llm_proxy/.env.example +4 -0
- smol_llm_proxy/__init__.py +3 -0
- smol_llm_proxy/__main__.py +12 -0
- smol_llm_proxy/auth.py +87 -0
- smol_llm_proxy/cache.py +59 -0
- smol_llm_proxy/config.example.yaml +13 -0
- smol_llm_proxy/config.py +13 -0
- smol_llm_proxy/config_loader.py +91 -0
- smol_llm_proxy/database.py +142 -0
- smol_llm_proxy/entrypoint.sh +4 -0
- smol_llm_proxy/main.py +320 -0
- smol_llm_proxy/metrics.py +232 -0
- smol_llm_proxy/proxy.py +227 -0
- smol_llm_proxy-0.1.0.dist-info/METADATA +268 -0
- smol_llm_proxy-0.1.0.dist-info/RECORD +19 -0
- smol_llm_proxy-0.1.0.dist-info/WHEEL +5 -0
- smol_llm_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- smol_llm_proxy-0.1.0.dist-info/licenses/LICENSE +201 -0
- smol_llm_proxy-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Entry point: python -m smol_llm_proxy."""
|
|
2
|
+
|
|
3
|
+
import uvicorn
|
|
4
|
+
from smol_llm_proxy.config import PROXY_HOST, PROXY_PORT
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def main():
|
|
8
|
+
uvicorn.run("smol_llm_proxy.main:app", host=PROXY_HOST, port=PROXY_PORT)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
if __name__ == "__main__":
|
|
12
|
+
main()
|
smol_llm_proxy/auth.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""API key management: CRUD and validation with SHA256 hashing."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import secrets
|
|
5
|
+
|
|
6
|
+
from .database import get_db
|
|
7
|
+
from .cache import get_cached_key, set_cached_key, clear_key_cache
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _row_to_dict(row) -> dict:
|
|
11
|
+
return dict(row) if row else None
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _hash_key(raw_key: str) -> str:
|
|
15
|
+
return hashlib.sha256(raw_key.encode()).hexdigest()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def create_api_key(name: str) -> dict:
|
|
19
|
+
"""Create a new API key. Returns dict with id, key, and name."""
|
|
20
|
+
raw_key = "sk-" + secrets.token_hex(24)
|
|
21
|
+
key_hash = _hash_key(raw_key)
|
|
22
|
+
|
|
23
|
+
with get_db() as conn:
|
|
24
|
+
cursor = conn.execute(
|
|
25
|
+
"INSERT INTO api_keys (key_hash, name) VALUES (?, ?)",
|
|
26
|
+
(key_hash, name),
|
|
27
|
+
)
|
|
28
|
+
key_id = cursor.lastrowid
|
|
29
|
+
clear_key_cache()
|
|
30
|
+
return {"id": key_id, "key": raw_key, "name": name}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _find_key_info(raw_key: str):
|
|
34
|
+
"""Find key info by direct hash lookup."""
|
|
35
|
+
key_hash = _hash_key(raw_key)
|
|
36
|
+
cached = get_cached_key(key_hash)
|
|
37
|
+
if cached:
|
|
38
|
+
return cached
|
|
39
|
+
|
|
40
|
+
with get_db() as conn:
|
|
41
|
+
row = conn.execute(
|
|
42
|
+
"SELECT id, key_hash, name, active FROM api_keys WHERE key_hash = ?",
|
|
43
|
+
(key_hash,),
|
|
44
|
+
).fetchone()
|
|
45
|
+
info = _row_to_dict(row)
|
|
46
|
+
if info:
|
|
47
|
+
set_cached_key(key_hash, info)
|
|
48
|
+
return info
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def delete_api_key(key_id: int) -> bool:
|
|
52
|
+
"""Delete an API key by its database id."""
|
|
53
|
+
with get_db() as conn:
|
|
54
|
+
cursor = conn.execute("DELETE FROM api_keys WHERE id = ?", (key_id,))
|
|
55
|
+
if cursor.rowcount > 0:
|
|
56
|
+
clear_key_cache()
|
|
57
|
+
return cursor.rowcount > 0
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def toggle_api_key(key_id: int, active: bool):
|
|
61
|
+
"""Activate or deactivate an API key."""
|
|
62
|
+
with get_db() as conn:
|
|
63
|
+
conn.execute(
|
|
64
|
+
"UPDATE api_keys SET active = ? WHERE id = ?",
|
|
65
|
+
(int(active), key_id),
|
|
66
|
+
)
|
|
67
|
+
row = conn.execute(
|
|
68
|
+
"SELECT id, name, active, created_at FROM api_keys WHERE id = ?",
|
|
69
|
+
(key_id,),
|
|
70
|
+
).fetchone()
|
|
71
|
+
clear_key_cache()
|
|
72
|
+
return _row_to_dict(row)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def list_api_keys() -> list[dict]:
|
|
76
|
+
"""List all API keys (never returns key hashes)."""
|
|
77
|
+
with get_db() as conn:
|
|
78
|
+
rows = conn.execute("SELECT id, name, active, created_at FROM api_keys ORDER BY created_at DESC").fetchall()
|
|
79
|
+
return [_row_to_dict(r) for r in rows]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def validate_api_key(raw_key: str):
|
|
83
|
+
"""Validate an API key by SHA256 hash lookup."""
|
|
84
|
+
info = _find_key_info(raw_key)
|
|
85
|
+
if info and info.get("active"):
|
|
86
|
+
return {"id": info["id"], "name": info["name"]}
|
|
87
|
+
return None
|
smol_llm_proxy/cache.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""In-memory caches for hot path optimization."""
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
_key_cache: dict[str, dict] = {}
|
|
8
|
+
_alias_cache: dict[str, str] = {}
|
|
9
|
+
_route_cache: dict[str, dict] = {}
|
|
10
|
+
|
|
11
|
+
TTL = 30
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_cached_key(key_hash: str) -> Optional[dict]:
|
|
15
|
+
entry = _key_cache.get(key_hash)
|
|
16
|
+
if entry and entry["_ts"] > time.time() - TTL:
|
|
17
|
+
return {k: v for k, v in entry.items() if k != "_ts"}
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def set_cached_key(key_hash: str, info: dict):
|
|
22
|
+
_key_cache[key_hash] = {**info, "_ts": time.time()}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def clear_key_cache():
|
|
26
|
+
_key_cache.clear()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def get_cached_alias(alias_name: str) -> Optional[str]:
|
|
30
|
+
return _alias_cache.get(alias_name)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def set_cached_alias(alias_name: str, real_model_name: str):
|
|
34
|
+
_alias_cache[alias_name] = real_model_name
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def clear_alias_cache():
|
|
38
|
+
_alias_cache.clear()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def get_cached_route(model_name: str) -> Optional[dict]:
|
|
42
|
+
entry = _route_cache.get(model_name)
|
|
43
|
+
if entry and entry["_ts"] > time.time() - TTL:
|
|
44
|
+
return {k: v for k, v in entry.items() if k != "_ts"}
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def set_cached_route(model_name: str, info: dict):
|
|
49
|
+
_route_cache[model_name] = {**info, "_ts": time.time()}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def clear_route_cache():
|
|
53
|
+
_route_cache.clear()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def clear_all():
|
|
57
|
+
clear_key_cache()
|
|
58
|
+
clear_alias_cache()
|
|
59
|
+
clear_route_cache()
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# smol-llm-proxy configuration
|
|
2
|
+
# Copy this file and fill in your llama-server URLs, models, and aliases
|
|
3
|
+
|
|
4
|
+
servers:
|
|
5
|
+
- name: my-server
|
|
6
|
+
url: http://host:port
|
|
7
|
+
api_key: "" # optional, if llama-server requires auth
|
|
8
|
+
models:
|
|
9
|
+
- model-name.gguf
|
|
10
|
+
|
|
11
|
+
# Aliases: short names → real model names
|
|
12
|
+
aliases:
|
|
13
|
+
alias: model-name.gguf
|
smol_llm_proxy/config.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Configuration for smol-llm-proxy."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
ADMIN_KEY: str = os.environ["ADMIN_KEY"]
|
|
7
|
+
PROXY_HOST: str = os.environ.get("PROXY_HOST", "0.0.0.0")
|
|
8
|
+
PROXY_PORT: int = int(os.environ.get("PROXY_PORT", "8000"))
|
|
9
|
+
HTTPX_TIMEOUT: float = float(os.environ.get("HTTPX_TIMEOUT", "120"))
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def get_db_path() -> Path:
|
|
13
|
+
return Path(os.environ.get("DB_PATH", "data/proxy.db"))
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Load and sync configuration from YAML file to database."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from .database import get_db
|
|
7
|
+
from .cache import set_cached_alias, set_cached_route
|
|
8
|
+
|
|
9
|
+
CONFIG_PATH = Path(os.environ.get("CONFIG_PATH", "config.yaml"))
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _load_yaml():
|
|
13
|
+
"""Load config.yaml, return parsed dict or empty dict."""
|
|
14
|
+
try:
|
|
15
|
+
import yaml
|
|
16
|
+
|
|
17
|
+
with open(CONFIG_PATH) as f:
|
|
18
|
+
return yaml.safe_load(f) or {}
|
|
19
|
+
except (FileNotFoundError, ImportError):
|
|
20
|
+
return {}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def sync_config():
|
|
24
|
+
"""Sync servers, models, and aliases from config.yaml into SQLite.
|
|
25
|
+
|
|
26
|
+
- Servers not in DB → created
|
|
27
|
+
- Models not assigned to server → assigned
|
|
28
|
+
- Aliases not in DB → created
|
|
29
|
+
Existing entries are left untouched (idempotent).
|
|
30
|
+
"""
|
|
31
|
+
cfg = _load_yaml()
|
|
32
|
+
|
|
33
|
+
servers = cfg.get("servers", [])
|
|
34
|
+
aliases = cfg.get("aliases", {})
|
|
35
|
+
|
|
36
|
+
with get_db() as conn:
|
|
37
|
+
# --- Servers ---
|
|
38
|
+
for srv in servers:
|
|
39
|
+
name = srv["name"]
|
|
40
|
+
url = srv["url"]
|
|
41
|
+
api_key = srv.get("api_key", "")
|
|
42
|
+
|
|
43
|
+
existing = conn.execute("SELECT id FROM servers WHERE name = ?", (name,)).fetchone()
|
|
44
|
+
|
|
45
|
+
if existing:
|
|
46
|
+
server_id = existing["id"]
|
|
47
|
+
# Update URL/api_key if changed
|
|
48
|
+
conn.execute(
|
|
49
|
+
"UPDATE servers SET url = ?, api_key = ? WHERE id = ?",
|
|
50
|
+
(url, api_key, server_id),
|
|
51
|
+
)
|
|
52
|
+
else:
|
|
53
|
+
cursor = conn.execute(
|
|
54
|
+
"INSERT INTO servers (name, url, api_key) VALUES (?, ?, ?)",
|
|
55
|
+
(name, url, api_key),
|
|
56
|
+
)
|
|
57
|
+
server_id = cursor.lastrowid
|
|
58
|
+
|
|
59
|
+
# --- Models for this server ---
|
|
60
|
+
for model_name in srv.get("models", []):
|
|
61
|
+
conn.execute(
|
|
62
|
+
"""INSERT INTO server_models (server_id, model_name)
|
|
63
|
+
SELECT ?, ? WHERE NOT EXISTS (
|
|
64
|
+
SELECT 1 FROM server_models WHERE server_id = ? AND model_name = ?)""",
|
|
65
|
+
(server_id, model_name, server_id, model_name),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
# --- Aliases ---
|
|
69
|
+
for alias_name, real_model in aliases.items():
|
|
70
|
+
conn.execute(
|
|
71
|
+
"""INSERT INTO model_aliases (alias_name, real_model_name)
|
|
72
|
+
SELECT ?, ? WHERE NOT EXISTS (
|
|
73
|
+
SELECT 1 FROM model_aliases WHERE alias_name = ?)""",
|
|
74
|
+
(alias_name, real_model, alias_name),
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
# Populate caches from DB
|
|
78
|
+
with get_db() as conn:
|
|
79
|
+
for row in conn.execute("SELECT alias_name, real_model_name FROM model_aliases").fetchall():
|
|
80
|
+
set_cached_alias(row["alias_name"], row["real_model_name"])
|
|
81
|
+
for row in conn.execute(
|
|
82
|
+
"""SELECT sm.model_name, s.url, s.api_key FROM server_models sm
|
|
83
|
+
JOIN servers s ON s.id = sm.server_id WHERE s.active = 1"""
|
|
84
|
+
).fetchall():
|
|
85
|
+
set_cached_route(row["model_name"], {"server_id": 0, "url": row["url"], "api_key": row["api_key"]})
|
|
86
|
+
|
|
87
|
+
# Populate key cache from DB
|
|
88
|
+
with get_db() as conn:
|
|
89
|
+
for row in conn.execute("SELECT id, key_hash, name, active FROM api_keys").fetchall():
|
|
90
|
+
pass
|
|
91
|
+
# We don't have raw key here, so we skip key cache population at startup
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""SQLite database schema and operations."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
import threading
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from contextlib import contextmanager
|
|
7
|
+
|
|
8
|
+
from .config import get_db_path
|
|
9
|
+
from .cache import get_cached_route, set_cached_route
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _row_to_dict(row):
|
|
13
|
+
return dict(row) if row else None
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
_thread_local = threading.local()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _get_connection(db_path: Path) -> sqlite3.Connection:
|
|
20
|
+
if not hasattr(_thread_local, "conn") or _thread_local.conn is None:
|
|
21
|
+
conn = sqlite3.connect(str(db_path))
|
|
22
|
+
conn.row_factory = sqlite3.Row
|
|
23
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
24
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
25
|
+
conn.execute("PRAGMA busy_timeout=5000")
|
|
26
|
+
_thread_local.conn = conn
|
|
27
|
+
return _thread_local.conn
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@contextmanager
|
|
31
|
+
def get_db():
|
|
32
|
+
conn = _get_connection(get_db_path())
|
|
33
|
+
try:
|
|
34
|
+
yield conn
|
|
35
|
+
conn.commit()
|
|
36
|
+
except Exception:
|
|
37
|
+
conn.rollback()
|
|
38
|
+
raise
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def reset_db_connection():
|
|
42
|
+
"""Close and reset the thread-local DB connection. For testing."""
|
|
43
|
+
if hasattr(_thread_local, "conn") and _thread_local.conn is not None:
|
|
44
|
+
try:
|
|
45
|
+
_thread_local.conn.close()
|
|
46
|
+
except Exception:
|
|
47
|
+
pass
|
|
48
|
+
_thread_local.conn = None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def validate_key(key_hash: str):
|
|
52
|
+
"""Check if a key is valid and active. Returns key info or None."""
|
|
53
|
+
with get_db() as conn:
|
|
54
|
+
row = conn.execute(
|
|
55
|
+
"SELECT id, name FROM api_keys WHERE key_hash = ? AND active = 1 LIMIT 1",
|
|
56
|
+
(key_hash,),
|
|
57
|
+
).fetchone()
|
|
58
|
+
return _row_to_dict(row)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def resolve_routing(key_id: int, model_name: str):
|
|
62
|
+
"""Resolve alias + find server for a given key. Returns server info or None."""
|
|
63
|
+
cache_key = f"{key_id}:{model_name}"
|
|
64
|
+
cached = get_cached_route(cache_key)
|
|
65
|
+
if cached:
|
|
66
|
+
return cached
|
|
67
|
+
|
|
68
|
+
with get_db() as conn:
|
|
69
|
+
row = conn.execute(
|
|
70
|
+
"""SELECT s.id as server_id, s.url, s.api_key,
|
|
71
|
+
COALESCE(ma.real_model_name, ?) as real_model
|
|
72
|
+
FROM api_keys ak
|
|
73
|
+
LEFT JOIN model_aliases ma ON ma.alias_name = ?
|
|
74
|
+
JOIN server_models sm ON sm.model_name = COALESCE(ma.real_model_name, ?)
|
|
75
|
+
JOIN servers s ON s.id = sm.server_id
|
|
76
|
+
WHERE ak.id = ? AND s.active = 1
|
|
77
|
+
LIMIT 1""",
|
|
78
|
+
(model_name, model_name, model_name, key_id),
|
|
79
|
+
).fetchone()
|
|
80
|
+
result = _row_to_dict(row)
|
|
81
|
+
if result:
|
|
82
|
+
set_cached_route(cache_key, result)
|
|
83
|
+
return result
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def init_db():
|
|
87
|
+
get_db_path().parent.mkdir(parents=True, exist_ok=True)
|
|
88
|
+
with get_db() as conn:
|
|
89
|
+
conn.executescript("""
|
|
90
|
+
CREATE TABLE IF NOT EXISTS servers (
|
|
91
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
92
|
+
name TEXT UNIQUE NOT NULL,
|
|
93
|
+
url TEXT NOT NULL,
|
|
94
|
+
api_key TEXT DEFAULT '',
|
|
95
|
+
active INTEGER NOT NULL DEFAULT 1
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
CREATE TABLE IF NOT EXISTS server_models (
|
|
99
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
100
|
+
server_id INTEGER NOT NULL REFERENCES servers(id) ON DELETE CASCADE,
|
|
101
|
+
model_name TEXT NOT NULL,
|
|
102
|
+
UNIQUE(server_id, model_name)
|
|
103
|
+
);
|
|
104
|
+
|
|
105
|
+
CREATE INDEX IF NOT EXISTS idx_server_models_model ON server_models(model_name);
|
|
106
|
+
|
|
107
|
+
CREATE TABLE IF NOT EXISTS model_aliases (
|
|
108
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
109
|
+
alias_name TEXT UNIQUE NOT NULL,
|
|
110
|
+
real_model_name TEXT NOT NULL
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
CREATE INDEX IF NOT EXISTS idx_model_aliases_alias ON model_aliases(alias_name);
|
|
114
|
+
|
|
115
|
+
CREATE TABLE IF NOT EXISTS api_keys (
|
|
116
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
117
|
+
key_hash TEXT UNIQUE NOT NULL,
|
|
118
|
+
name TEXT NOT NULL,
|
|
119
|
+
active INTEGER NOT NULL DEFAULT 1,
|
|
120
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
121
|
+
);
|
|
122
|
+
|
|
123
|
+
CREATE INDEX IF NOT EXISTS idx_api_keys_hash ON api_keys(key_hash);
|
|
124
|
+
|
|
125
|
+
CREATE TABLE IF NOT EXISTS usage_logs (
|
|
126
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
127
|
+
key_id INTEGER NOT NULL REFERENCES api_keys(id) ON DELETE CASCADE,
|
|
128
|
+
server_id INTEGER NOT NULL REFERENCES servers(id) ON DELETE CASCADE,
|
|
129
|
+
model_name TEXT NOT NULL,
|
|
130
|
+
real_model_name TEXT NOT NULL DEFAULT '',
|
|
131
|
+
prompt_tokens INTEGER NOT NULL DEFAULT 0,
|
|
132
|
+
completion_tokens INTEGER NOT NULL DEFAULT 0,
|
|
133
|
+
total_tokens INTEGER NOT NULL DEFAULT 0,
|
|
134
|
+
prompt_ms REAL DEFAULT 0,
|
|
135
|
+
predicted_ms REAL DEFAULT 0,
|
|
136
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
CREATE INDEX IF NOT EXISTS idx_usage_logs_key ON usage_logs(key_id);
|
|
140
|
+
CREATE INDEX IF NOT EXISTS idx_usage_logs_server ON usage_logs(server_id);
|
|
141
|
+
CREATE INDEX IF NOT EXISTS idx_usage_logs_created ON usage_logs(created_at);
|
|
142
|
+
""")
|