freelm 0.1.0__py3-none-any.whl

freelm/__init__.py

@@ -0,0 +1,63 @@
+"""freelm — one always-up LLM client over free-tier providers.
+
+Quick start::
+
+    import freelm
+    llm = freelm.FreeLLM.from_env()
+    print(llm.text("Explain black holes in one sentence."))
+
+Explicit config::
+
+    from freelm import FreeLLM, OpenRouter, GoogleAIStudio, NIM
+    llm = FreeLLM(
+        providers=[OpenRouter("sk-or-..."), GoogleAIStudio("AIza..."), NIM("nvapi-...")],
+        strategy="quota_aware",
+    )
+"""
+from __future__ import annotations
+
+from ._types import ChatRequest, ChatResponse, Choice, Message, Usage
+from .client import AsyncFreeLLM, FreeLLM
+from .config import providers_from_env
+from .discovery import list_free_models
+from .errors import (
+    AuthError,
+    ConfigError,
+    FreeLLMError,
+    ModelNotFound,
+    NoProvidersAvailable,
+    ProviderError,
+    RateLimited,
+    Transient,
+)
+from .providers import Gemini, GoogleAIStudio, NIM, OpenRouter, Provider
+from .registry import ModelSpec
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "FreeLLM",
+    "AsyncFreeLLM",
+    "Provider",
+    "OpenRouter",
+    "GoogleAIStudio",
+    "Gemini",
+    "NIM",
+    "ModelSpec",
+    "Message",
+    "ChatRequest",
+    "ChatResponse",
+    "Choice",
+    "Usage",
+    "providers_from_env",
+    "list_free_models",
+    "FreeLLMError",
+    "ConfigError",
+    "ProviderError",
+    "AuthError",
+    "RateLimited",
+    "Transient",
+    "ModelNotFound",
+    "NoProvidersAvailable",
+    "__version__",
+]

freelm/_backoff.py

@@ -0,0 +1,13 @@
+"""Exponential backoff with full jitter."""
+from __future__ import annotations
+
+import random
+
+
+def compute_delay(attempt: int, base: float = 0.5, factor: float = 2.0, cap: float = 30.0, jitter: bool = True) -> float:
+    """Delay (seconds) for a given retry attempt (1-based)."""
+    attempt = max(1, attempt)
+    raw = min(cap, base * (factor ** (attempt - 1)))
+    if jitter:
+        return random.uniform(0.0, raw)
+    return raw

freelm/_breaker.py

@@ -0,0 +1,41 @@
+"""Per-key circuit breaker. Time is injected (monotonic seconds) for testability."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+CLOSED = "closed"
+OPEN = "open"
+HALF_OPEN = "half_open"
+
+
+@dataclass
+class CircuitBreaker:
+    fail_threshold: int = 4
+    cooldown: float = 30.0
+    state: str = CLOSED
+    failures: int = 0
+    opened_at: float = 0.0
+
+    def allow(self, now: float) -> bool:
+        """May a request go through right now?"""
+        if self.state == OPEN:
+            if now - self.opened_at >= self.cooldown:
+                self.state = HALF_OPEN
+                return True
+            return False
+        return True
+
+    def on_success(self) -> None:
+        self.failures = 0
+        self.state = CLOSED
+
+    def on_failure(self, now: float) -> None:
+        self.failures += 1
+        if self.state == HALF_OPEN or self.failures >= self.fail_threshold:
+            self.state = OPEN
+            self.opened_at = now
+
+    def time_until_half_open(self, now: float) -> float:
+        if self.state != OPEN:
+            return 0.0
+        return max(0.0, self.cooldown - (now - self.opened_at))

freelm/_cache.py

@@ -0,0 +1,62 @@
+"""Tiny TTL disk cache for discovered model lists.
+
+Mirrors the openrouter-free skill: JSON file, TTL (default 1h, env override),
+0o600 perms. Path: $FREELM_CACHE_DIR or ~/.cache/freelm/models-<provider>.json
+"""
+from __future__ import annotations
+
+import json
+import os
+import time
+from typing import Any, List, Optional
+
+DEFAULT_TTL = 3600.0
+
+
+def cache_dir() -> str:
+    return os.environ.get("FREELM_CACHE_DIR") or os.path.join(os.path.expanduser("~"), ".cache", "freelm")
+
+
+def default_ttl() -> float:
+    raw = os.environ.get("FREELM_CACHE_TTL")
+    if not raw:
+        return DEFAULT_TTL
+    try:
+        return float(raw)
+    except ValueError:
+        return DEFAULT_TTL
+
+
+def _path(name: str) -> str:
+    safe = name.replace("/", "_")
+    return os.path.join(cache_dir(), f"models-{safe}.json")
+
+
+def load(name: str) -> Optional[List[Any]]:
+    try:
+        with open(_path(name), "r", encoding="utf-8") as f:
+            entry = json.load(f)
+        if time.time() > entry.get("expires_at", 0):
+            return None
+        return entry.get("data")
+    except (OSError, ValueError):
+        return None
+
+
+def save(name: str, data: List[Any], ttl: Optional[float] = None) -> None:
+    p = _path(name)
+    try:
+        os.makedirs(os.path.dirname(p), exist_ok=True)
+        entry = {"data": data, "expires_at": time.time() + (ttl if ttl is not None else default_ttl())}
+        with open(p, "w", encoding="utf-8") as f:
+            json.dump(entry, f)
+        os.chmod(p, 0o600)
+    except OSError:
+        pass  # cache is best-effort; never fatal
+
+
+def clear(name: str) -> None:
+    try:
+        os.remove(_path(name))
+    except OSError:
+        pass

freelm/_engine.py

@@ -0,0 +1,87 @@
+"""Pure (no-I/O) orchestration helpers shared by the sync and async clients.
+
+The actual HTTP call differs between sync/async, but candidate selection and
+post-attempt state updates are identical, so they live here.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+from ._backoff import compute_delay
+from .errors import AuthError, ModelNotFound, ProviderError, RateLimited, Transient
+from .strategy import Candidate, order_candidates
+
+TriedKey = Tuple[str, str, str]
+
+
+def select_candidate(
+    providers: List[Any],
+    strategy: str,
+    rr: Dict[str, int],
+    alias: str,
+    tried: Set[TriedKey],
+    now: float,
+) -> Optional[Candidate]:
+    """First ready candidate not already tried this call, in strategy order."""
+    for c in order_candidates(providers, alias, now, strategy, rr):
+        if (c.provider.name, c.key.key, c.model) in tried:
+            continue
+        if c.key.ready(now):
+            return c
+    return None
+
+
+def soonest_wait(providers: List[Any], now: float) -> Optional[float]:
+    """Smallest wait until *some* non-disabled key becomes ready, or None."""
+    waits: List[float] = []
+    for p in providers:
+        for k in p.keys:
+            w = k.wait_time(now)
+            if w is not None:
+                waits.append(w)
+    return min(waits) if waits else None
+
+
+def apply_success(cand: Candidate, latency_ms: float) -> None:
+    k = cand.key
+    k.breaker.on_success()
+    k.last_error = None
+    k.ewma_latency = latency_ms if k.ewma_latency == 0 else 0.7 * k.ewma_latency + 0.3 * latency_ms
+
+
+def apply_error(cand: Candidate, exc: ProviderError, now: float) -> None:
+    """Update key state after a failed attempt. Returns nothing; raising is the
+    caller's decision (see ``should_raise``)."""
+    k = cand.key
+    if isinstance(exc, AuthError):
+        k.disabled = True
+        k.last_error = f"auth:{exc.status}"
+    elif isinstance(exc, RateLimited):
+        if getattr(exc, "scope", "key") == "model":
+            # only this model is throttled upstream — keep the key hot, the
+            # 'tried' set steers us to the next model on the same key.
+            k.last_error = "rate_limited:model"
+        else:
+            k.cooldown_until = now + (exc.retry_after or 60.0)
+            k.last_error = "rate_limited"
+    elif isinstance(exc, ModelNotFound):
+        k.last_error = "model_missing"  # don't penalise the key for a bad model id
+    elif isinstance(exc, Transient):
+        k.breaker.on_failure(now)
+        delay = exc.retry_after if exc.retry_after is not None else compute_delay(k.breaker.failures)
+        k.cooldown_until = now + min(30.0, delay)
+        k.last_error = f"transient:{exc.status}"
+    else:  # non-retryable ProviderError
+        k.breaker.on_failure(now)
+        k.last_error = f"error:{exc.status}"
+
+
+def should_raise(exc: ProviderError) -> bool:
+    """A non-retryable, non-model error (e.g. malformed 400/422) is a caller bug:
+    bail immediately instead of burning every key on the same broken request.
+
+    Auth errors are *not* fatal to the whole call — the key is disabled and we
+    fail over to other keys/providers."""
+    if isinstance(exc, AuthError):
+        return False
+    return not exc.retryable and not exc.model_missing

freelm/_keys.py

@@ -0,0 +1,94 @@
+"""Per-key runtime state: breaker + rpm bucket + daily quota + cooldowns."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+from ._breaker import CircuitBreaker
+from ._ratelimit import TokenBucket
+
+DAY = 86400.0
+
+
+@dataclass
+class KeyState:
+    key: str
+    tier: str = "free"
+    breaker: CircuitBreaker = field(default_factory=CircuitBreaker)
+    bucket: Optional[TokenBucket] = None
+    rpd: Optional[int] = None           # requests-per-day cap (None = unknown/unlimited)
+    rpd_used: int = 0
+    rpd_reset: float = 0.0              # monotonic ts at which the daily counter rolls
+    cooldown_until: float = 0.0
+    disabled: bool = False              # hard-off after auth failure
+    ewma_latency: float = 0.0
+    last_error: Optional[str] = None
+
+    # -- daily window ----------------------------------------------------
+    def _roll_daily(self, now: float) -> None:
+        if self.rpd is None:
+            return
+        if self.rpd_reset == 0.0:
+            self.rpd_reset = now + DAY
+        elif now >= self.rpd_reset:
+            self.rpd_used = 0
+            self.rpd_reset = now + DAY
+
+    # -- gating ----------------------------------------------------------
+    def ready(self, now: float) -> bool:
+        if self.disabled:
+            return False
+        if now < self.cooldown_until:
+            return False
+        if not self.breaker.allow(now):
+            return False
+        self._roll_daily(now)
+        if self.rpd is not None and self.rpd_used >= self.rpd:
+            return False
+        if self.bucket is not None and self.bucket.peek(now) < 1:
+            return False
+        return True
+
+    def reserve(self, now: float) -> bool:
+        """Consume one rpm token + one daily slot just before firing a request."""
+        self._roll_daily(now)
+        if self.bucket is not None and not self.bucket.consume(1, now):
+            return False
+        self.rpd_used += 1
+        return True
+
+    def remaining(self, now: float) -> float:
+        """A rough 'how much headroom' score for quota-aware routing."""
+        self._roll_daily(now)
+        daily = float("inf") if self.rpd is None else float(max(0, self.rpd - self.rpd_used))
+        burst = self.bucket.peek(now) if self.bucket is not None else float("inf")
+        return min(daily, burst)
+
+    def wait_time(self, now: float) -> Optional[float]:
+        """Seconds until this key could be ready again, or None if permanently off."""
+        if self.disabled:
+            return None
+        waits = []
+        if now < self.cooldown_until:
+            waits.append(self.cooldown_until - now)
+        waits.append(self.breaker.time_until_half_open(now))
+        self._roll_daily(now)
+        if self.rpd is not None and self.rpd_used >= self.rpd:
+            waits.append(max(0.0, self.rpd_reset - now))
+        if self.bucket is not None and self.bucket.peek(now) < 1:
+            waits.append(self.bucket.time_until(1, now))
+        return max(waits) if waits else 0.0
+
+    def masked(self) -> str:
+        k = self.key
+        return (k[:6] + "..." + k[-4:]) if len(k) > 12 else "***"
+
+
+def new_key_state(key: str, *, tier: str, rpm: Optional[float], rpd: Optional[int]) -> KeyState:
+    return KeyState(
+        key=key,
+        tier=tier,
+        breaker=CircuitBreaker(),
+        bucket=TokenBucket(rate_per_min=rpm) if rpm else None,
+        rpd=rpd,
+    )

freelm/_ratelimit.py

@@ -0,0 +1,52 @@
+"""Token bucket for requests-per-minute pacing. Time injected for testability."""
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class TokenBucket:
+    rate_per_min: float
+    capacity: Optional[float] = None
+    tokens: Optional[float] = None
+    updated: float = 0.0
+    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False, compare=False)
+
+    def __post_init__(self) -> None:
+        if self.capacity is None:
+            self.capacity = max(1.0, float(self.rate_per_min))
+        if self.tokens is None:
+            self.tokens = self.capacity
+
+    def _refill(self, now: float) -> None:
+        if self.updated == 0.0:
+            self.updated = now
+            return
+        dt = now - self.updated
+        if dt <= 0:
+            return
+        self.tokens = min(self.capacity, self.tokens + dt * (self.rate_per_min / 60.0))  # type: ignore[operator]
+        self.updated = now
+
+    def peek(self, now: float) -> float:
+        with self._lock:
+            self._refill(now)
+            return self.tokens  # type: ignore[return-value]
+
+    def consume(self, n: float, now: float) -> bool:
+        with self._lock:
+            self._refill(now)
+            if self.tokens >= n:  # type: ignore[operator]
+                self.tokens -= n  # type: ignore[operator]
+                return True
+            return False
+
+    def time_until(self, n: float, now: float) -> float:
+        with self._lock:
+            self._refill(now)
+            if self.tokens >= n:  # type: ignore[operator]
+                return 0.0
+            deficit = n - self.tokens  # type: ignore[operator]
+            return deficit / (self.rate_per_min / 60.0)

freelm/_types.py

@@ -0,0 +1,119 @@
+"""Provider-agnostic data types (OpenAI-shaped, but pure dataclasses)."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Union
+
+
+@dataclass
+class Message:
+    role: str
+    content: Optional[str] = None
+    name: Optional[str] = None
+    tool_calls: Optional[List[Dict[str, Any]]] = None
+    tool_call_id: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        d: Dict[str, Any] = {"role": self.role}
+        if self.content is not None:
+            d["content"] = self.content
+        if self.name:
+            d["name"] = self.name
+        if self.tool_calls:
+            d["tool_calls"] = self.tool_calls
+        if self.tool_call_id:
+            d["tool_call_id"] = self.tool_call_id
+        return d
+
+    @classmethod
+    def from_any(cls, m: Union["Message", Dict[str, Any], str]) -> "Message":
+        if isinstance(m, Message):
+            return m
+        if isinstance(m, str):
+            return cls(role="user", content=m)
+        if isinstance(m, dict):
+            return cls(
+                role=m.get("role", "user"),
+                content=m.get("content"),
+                name=m.get("name"),
+                tool_calls=m.get("tool_calls"),
+                tool_call_id=m.get("tool_call_id"),
+            )
+        raise TypeError(f"unsupported message type: {type(m)!r}")
+
+
+@dataclass
+class Usage:
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_tokens: int = 0
+
+    @classmethod
+    def from_dict(cls, d: Optional[Dict[str, Any]]) -> "Usage":
+        d = d or {}
+        return cls(
+            prompt_tokens=int(d.get("prompt_tokens") or 0),
+            completion_tokens=int(d.get("completion_tokens") or 0),
+            total_tokens=int(d.get("total_tokens") or 0),
+        )
+
+
+@dataclass
+class Choice:
+    index: int
+    message: Message
+    finish_reason: Optional[str] = None
+
+
+@dataclass
+class ChatResponse:
+    id: Optional[str]
+    model: Optional[str]
+    provider: Optional[str]
+    choices: List[Choice]
+    usage: Usage
+    latency_ms: float = 0.0
+    raw: Optional[Dict[str, Any]] = None
+
+    @property
+    def text(self) -> str:
+        if not self.choices:
+            return ""
+        return self.choices[0].message.content or ""
+
+    def __str__(self) -> str:  # so print(resp) gives the text
+        return self.text
+
+
+_SAMPLING_FIELDS = ("temperature", "max_tokens", "top_p", "stop", "seed", "frequency_penalty", "presence_penalty")
+
+
+@dataclass
+class ChatRequest:
+    messages: List[Dict[str, Any]]
+    model: str = "auto"          # virtual alias resolved per provider
+    temperature: Optional[float] = None
+    max_tokens: Optional[int] = None
+    top_p: Optional[float] = None
+    stop: Optional[Union[str, List[str]]] = None
+    seed: Optional[int] = None
+    frequency_penalty: Optional[float] = None
+    presence_penalty: Optional[float] = None
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+    def payload(self, concrete_model: str) -> Dict[str, Any]:
+        body: Dict[str, Any] = {"model": concrete_model, "messages": self.messages}
+        for k in _SAMPLING_FIELDS:
+            v = getattr(self, k)
+            if v is not None:
+                body[k] = v
+        body.update(self.extra)
+        return body
+
+
+def build_request(messages: Any, model: str, kw: Dict[str, Any]) -> ChatRequest:
+    if not isinstance(messages, (list, tuple)):
+        messages = [messages]
+    msgs = [Message.from_any(m).to_dict() for m in messages]
+    fields = {k: kw.pop(k) for k in list(kw) if k in _SAMPLING_FIELDS}
+    return ChatRequest(messages=msgs, model=model, extra=dict(kw), **fields)