synckey 0.1.0__py3-none-any.whl

synckey/deferred.py

@@ -0,0 +1,41 @@
+"""Deferred request queue helpers.
+
+When every key for a request is cooling, the gateway can't serve it now. Instead
+of a 503, it persists the request, hands back an id + ETA, and a background worker
+replays it through the normal keypool once capacity frees. The result is stored
+and served on a later GET, then purged `deferred_ttl` seconds after completion.
+
+This module holds the small pure helpers; the worker loop and the run path live
+in server.py (they reuse its forwarding logic).
+"""
+
+from __future__ import annotations
+
+import secrets
+
+DEFER_PREFIX = "defer_"
+
+
+def new_request_id() -> str:
+    return DEFER_PREFIX + secrets.token_urlsafe(18)
+
+
+def estimate_eta(ctx, providers: list[str]) -> float:
+    """Seconds until the soonest key for these providers is expected to be usable.
+
+    Accounts for both reasons a key can be unavailable: a provider cooldown (use
+    its remaining window) and a drained local bucket (use its refill time). Falls
+    back to the configured default cooldown when nothing reports a wait.
+    """
+    waits: list[float] = []
+    for pid in providers:
+        for cand in ctx.pool.candidates(pid):
+            if cand.ready:
+                return 0.0  # something can already go (rare at defer time)
+            if cand.cooling and cand.cooldown_remaining > 0:
+                waits.append(cand.cooldown_remaining)
+            else:
+                waits.append(cand.bucket.seconds_to_capacity())
+    if waits:
+        return max(0.0, min(waits))
+    return float(ctx.settings.default_cooldown)

synckey/keypool.py

@@ -0,0 +1,139 @@
+"""Key pool: proactive + tier-aware selection.
+
+Selection order for a provider:
+  1. LIVE keys whose bucket has capacity  (best: no 429 risk)
+  2. LIVE keys whose bucket is exhausted  (might 429 but key itself is healthy)
+  3. COOLING keys, soonest-free first     (last resort)
+  DEAD keys are never returned.
+
+This means the gateway stops sending to a key *before* it 429s, not after.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+
+from .bucket import Bucket, BucketRegistry
+from .crypto import SecretBox
+from .db import Database, KeyRecord
+from .state import Health, StateStore
+
+
+@dataclass
+class Candidate:
+    key_id: int
+    label: str
+    secret: str
+    provider: str
+    ready: bool          # True if bucket has capacity right now
+    cooling: bool
+    cooldown_remaining: float
+    bucket: Bucket
+
+
+def parse_retry_after(value: str | None) -> float | None:
+    if not value:
+        return None
+    try:
+        return float(value)
+    except ValueError:
+        return None
+
+
+class KeyPool:
+    def __init__(
+        self,
+        db: Database,
+        box: SecretBox,
+        states: StateStore,
+        default_cooldown: float = 20.0,
+    ):
+        self.db = db
+        self.box = box
+        self.states = states
+        self.default_cooldown = default_cooldown
+        self._secrets: dict[int, str] = {}
+        self._buckets = BucketRegistry()
+        self._cursor: dict[str, int] = {}
+
+    def _secret(self, rec: KeyRecord) -> str:
+        s = self._secrets.get(rec.id)
+        if s is None:
+            s = self.box.open(rec.secret)
+            self._secrets[rec.id] = s
+        return s
+
+    def candidates(self, provider: str, estimated_tokens: int = 0) -> list[Candidate]:
+        records = self.db.list_keys(provider=provider, enabled_only=True)
+        if not records:
+            return []
+
+        weighted: list[KeyRecord] = []
+        for r in records:
+            weighted.extend([r] * max(1, r.weight))
+
+        start = self._cursor.get(provider, 0) % len(weighted)
+        self._cursor[provider] = start + 1
+        rotated = weighted[start:] + weighted[:start]
+
+        now = time.time()
+        ready: list[Candidate] = []
+        throttled: list[Candidate] = []
+        cooling: list[Candidate] = []
+        seen: set[int] = set()
+
+        for r in rotated:
+            if r.id in seen:
+                continue
+            seen.add(r.id)
+
+            st = self.states.get(r.id)
+            if st.health == Health.DEAD:
+                continue
+
+            bucket = self._buckets.get(r.id, rpm_cap=r.rpm_limit, tpm_cap=r.tpm_limit)
+            remaining = max(0.0, st.cooldown_until - now) if st.health == Health.COOLING else 0.0
+            is_cooling = remaining > 0
+            can_send = bucket.can_send(estimated_tokens)
+
+            cand = Candidate(
+                key_id=r.id,
+                label=r.label,
+                secret=self._secret(r),
+                provider=provider,
+                ready=can_send and not is_cooling,
+                cooling=is_cooling,
+                cooldown_remaining=remaining,
+                bucket=bucket,
+            )
+            if is_cooling:
+                cooling.append(cand)
+            elif can_send:
+                ready.append(cand)
+            else:
+                throttled.append(cand)
+
+        cooling.sort(key=lambda c: c.cooldown_remaining)
+        return ready + throttled + cooling
+
+    def on_success(self, key_id: int) -> None:
+        self.states.set_live(key_id)
+        self._buckets.get(key_id).on_success()
+
+    def on_rate_limit(self, key_id: int, retry_after: float | None, rpm_at_429: float | None = None) -> None:
+        cooldown = retry_after if retry_after and retry_after > 0 else self.default_cooldown
+        self.states.set_cooling(key_id, time.time() + cooldown, rpm_observed=rpm_at_429)
+        self._buckets.get(key_id).on_rate_limit(retry_after, rpm_at_429)
+
+    def on_dead(self, key_id: int, reason: str) -> None:
+        self.states.set_dead(key_id, reason)
+
+    def on_failure(self, key_id: int) -> None:
+        st = self.states.get(key_id)
+        fails = (1 if st.health == Health.LIVE else 2)
+        delay = min(self.default_cooldown, 2.0 * fails)
+        self.states.set_cooling(key_id, time.time() + delay)
+
+    def consume(self, key_id: int, estimated_tokens: int = 0) -> None:
+        self._buckets.get(key_id).consume(estimated_tokens)

synckey/prefs.py

@@ -0,0 +1,82 @@
+"""User-managed routing preferences.
+
+These are the knobs a user sets interactively (via `synckey setup`, `config`,
+`tier`, or the `key add` flow) rather than hand-editing a file:
+
+    global_default_model   model used when a request names neither model nor provider
+    provider_defaults      {provider: model} used when a request names only a provider
+    aliases                {name: target}   target is a model id or a tier name
+    tier_overrides         {model: tier}    manual tier assignment
+
+They live in DB meta (JSON) so they survive restarts and are editable
+programmatically. config.toml stays the home for *static* infra (host/port,
+custom providers, price overrides) that you version-control; these are the
+*dynamic* choices you tune as you go. `synckey config` shows both in one place.
+"""
+
+from __future__ import annotations
+
+from .db import Database
+
+GLOBAL_DEFAULT = "global_default_model"
+PROVIDER_DEFAULTS = "provider_defaults"
+ALIASES = "aliases"
+TIER_OVERRIDES = "tier_overrides_db"
+
+
+class Prefs:
+    def __init__(self, db: Database):
+        self.db = db
+
+    # global default
+    def global_default(self) -> str | None:
+        return self.db.get_meta(GLOBAL_DEFAULT)
+
+    def set_global_default(self, model: str) -> None:
+        self.db.set_meta(GLOBAL_DEFAULT, model)
+
+    # per-provider defaults
+    def provider_defaults(self) -> dict[str, str]:
+        return self.db.get_json(PROVIDER_DEFAULTS, {}) or {}
+
+    def provider_default(self, provider: str) -> str | None:
+        return self.provider_defaults().get(provider.lower())
+
+    def set_provider_default(self, provider: str, model: str) -> None:
+        d = self.provider_defaults()
+        d[provider.lower()] = model
+        self.db.set_json(PROVIDER_DEFAULTS, d)
+
+    # aliases
+    def aliases(self) -> dict[str, str]:
+        return self.db.get_json(ALIASES, {}) or {}
+
+    def set_alias(self, name: str, target: str) -> None:
+        a = self.aliases()
+        a[name.lower()] = target
+        self.db.set_json(ALIASES, a)
+
+    def remove_alias(self, name: str) -> bool:
+        a = self.aliases()
+        if name.lower() in a:
+            del a[name.lower()]
+            self.db.set_json(ALIASES, a)
+            return True
+        return False
+
+    # tier overrides
+    def tier_overrides(self) -> dict[str, str]:
+        return self.db.get_json(TIER_OVERRIDES, {}) or {}
+
+    def set_tier_override(self, model: str, tier: str) -> None:
+        t = self.tier_overrides()
+        t[model.lower()] = tier
+        self.db.set_json(TIER_OVERRIDES, t)
+
+    def remove_tier_override(self, model: str) -> bool:
+        t = self.tier_overrides()
+        if model.lower() in t:
+            del t[model.lower()]
+            self.db.set_json(TIER_OVERRIDES, t)
+            return True
+        return False

synckey/providers.py

@@ -0,0 +1,189 @@
+"""Built-in provider registry.
+
+Every provider exposes an OpenAI-compatible REST surface, so one adapter (base
+URL plus a bearer key) reaches all of them. Patterns are compiled once at
+construction and only used as a fallback when prefix and live-index routing miss.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Pattern
+
+
+@dataclass
+class Provider:
+    id: str
+    name: str
+    base_url: str
+    models_path: str = "/models"
+    env: tuple[str, ...] = ()
+    patterns: tuple[str, ...] = ()
+    auth: str = "bearer"  # "bearer" | "x-api-key" | "query:<param>"
+    signup: str = ""
+    notes: str = ""
+    _rx: list[Pattern[str]] = field(default_factory=list, init=False, repr=False, compare=False)
+
+    def __post_init__(self) -> None:
+        self._rx = [re.compile(p, re.IGNORECASE) for p in self.patterns]
+
+    def matches(self, model: str) -> bool:
+        return any(rx.search(model) for rx in self._rx)
+
+    def auth_headers(self, secret: str) -> dict[str, str]:
+        if self.auth == "bearer":
+            return {"Authorization": f"Bearer {secret}"}
+        if self.auth == "x-api-key":
+            return {"x-api-key": secret}
+        return {}
+
+    def auth_params(self, secret: str) -> dict[str, str]:
+        if self.auth.startswith("query:"):
+            return {self.auth.split(":", 1)[1]: secret}
+        return {}
+
+
+# Patterns only claim models a provider unambiguously owns. Shared open-weight
+# names (llama, qwen, deepseek) are resolved at runtime via the live index.
+BUILTIN: dict[str, Provider] = {
+    p.id: p
+    for p in (
+        Provider(
+            id="gemini",
+            name="Google Gemini",
+            base_url="https://generativelanguage.googleapis.com/v1beta/openai",
+            env=("GEMINI_API_KEY", "GOOGLE_API_KEY"),
+            patterns=(r"^gemini[-/]", r"^gemma[-/]?", r"^learnlm", r"^text-embedding-00"),
+            signup="https://aistudio.google.com/apikey",
+        ),
+        Provider(
+            id="groq",
+            name="Groq",
+            base_url="https://api.groq.com/openai/v1",
+            env=("GROQ_API_KEY",),
+            patterns=(r"versatile$", r"instant$", r"^whisper-large", r"^groq/", r"^gpt-oss", r"^openai/gpt-oss"),
+            signup="https://console.groq.com/keys",
+        ),
+        Provider(
+            id="cerebras",
+            name="Cerebras",
+            base_url="https://api.cerebras.ai/v1",
+            env=("CEREBRAS_API_KEY",),
+            patterns=(r"^llama3\.1-", r"^llama-3\.3-70b$", r"^cerebras/"),
+            signup="https://cloud.cerebras.ai",
+        ),
+        Provider(
+            id="nvidia",
+            name="NVIDIA NIM",
+            base_url="https://integrate.api.nvidia.com/v1",
+            env=("NVIDIA_API_KEY", "NVIDIA_NIM_API_KEY"),
+            patterns=(r"^nvidia/", r"nemotron"),
+            signup="https://build.nvidia.com",
+            notes="Models are org-namespaced, e.g. meta/llama-3.3-70b-instruct.",
+        ),
+        Provider(
+            id="github",
+            name="GitHub Models",
+            base_url="https://models.github.ai/inference",
+            models_path="/catalog/models",
+            env=("GITHUB_TOKEN", "GITHUB_MODELS_TOKEN"),
+            patterns=(r"^github/",),
+            signup="https://github.com/settings/personal-access-tokens",
+            notes="Use a fine-grained PAT with the models:read permission.",
+        ),
+        Provider(
+            id="cohere",
+            name="Cohere",
+            base_url="https://api.cohere.ai/compatibility/v1",
+            env=("COHERE_API_KEY", "CO_API_KEY"),
+            patterns=(r"^command", r"^embed-", r"^rerank", r"^cohere/"),
+            signup="https://dashboard.cohere.com/api-keys",
+        ),
+        Provider(
+            id="openrouter",
+            name="OpenRouter",
+            base_url="https://openrouter.ai/api/v1",
+            env=("OPENROUTER_API_KEY",),
+            patterns=(r"^openrouter/",),
+            signup="https://openrouter.ai/keys",
+        ),
+        Provider(
+            id="together",
+            name="Together AI",
+            base_url="https://api.together.xyz/v1",
+            env=("TOGETHER_API_KEY",),
+            patterns=(r"^together/",),
+            signup="https://api.together.ai/settings/api-keys",
+        ),
+        Provider(
+            id="mistral",
+            name="Mistral AI",
+            base_url="https://api.mistral.ai/v1",
+            env=("MISTRAL_API_KEY",),
+            patterns=(r"^mistral", r"^codestral", r"^ministral", r"^pixtral", r"^magistral"),
+            signup="https://console.mistral.ai/api-keys",
+        ),
+        Provider(
+            id="deepseek",
+            name="DeepSeek",
+            base_url="https://api.deepseek.com/v1",
+            env=("DEEPSEEK_API_KEY",),
+            patterns=(r"^deepseek-chat$", r"^deepseek-reasoner$"),
+            signup="https://platform.deepseek.com/api_keys",
+        ),
+        Provider(
+            id="xai",
+            name="xAI Grok",
+            base_url="https://api.x.ai/v1",
+            env=("XAI_API_KEY",),
+            patterns=(r"^grok",),
+            signup="https://console.x.ai",
+        ),
+        Provider(
+            id="sambanova",
+            name="SambaNova",
+            base_url="https://api.sambanova.ai/v1",
+            env=("SAMBANOVA_API_KEY",),
+            patterns=(r"^sambanova/",),
+            signup="https://cloud.sambanova.ai/apis",
+        ),
+        Provider(
+            id="openai",
+            name="OpenAI",
+            base_url="https://api.openai.com/v1",
+            env=("OPENAI_API_KEY",),
+            patterns=(r"^gpt-", r"^o[0-9]", r"^chatgpt", r"^dall-e", r"^text-embedding-3"),
+            signup="https://platform.openai.com/api-keys",
+        ),
+        Provider(
+            id="ollama",
+            name="Ollama (local)",
+            base_url="http://localhost:11434/v1",
+            env=("OLLAMA_HOST",),
+            patterns=(r"^ollama/",),
+            signup="https://ollama.com",
+            notes="No key required for a default local install.",
+        ),
+    )
+}
+
+
+def split_provider_prefix(model: str, known_ids: set[str]) -> tuple[str | None, str]:
+    """Split a provider/model prefix when the head is a registered provider.
+
+    Org-namespaced names like meta/llama-3.3-70b are left intact because meta is
+    not a registered provider.
+    """
+    if "/" in model:
+        head, tail = model.split("/", 1)
+        if head.lower() in known_ids:
+            return head.lower(), tail
+    return None, model
+
+
+def match_by_pattern(model: str, providers: dict[str, Provider]) -> str | None:
+    for pid, prov in providers.items():
+        if prov.matches(model):
+            return pid
+    return None