simplyllm 0.1.0__py3-none-any.whl

myllm/__init__.py

@@ -0,0 +1,30 @@
+"""myllm - Free AI LLM client with multi-provider fallback.
+
+Provides a unified API for multiple AI providers (Cerebras, Groq,
+OpenRouter, NVIDIA NIM) with automatic fallback, rate limiting, and
+cooldown tracking.
+
+Example:
+    >>> from myllm import MyLLM
+    >>> client = MyLLM(server_url="http://localhost:10000")
+    >>> result = client.chat([{"role": "user", "content": "Hello"}])
+    >>> print(result.content, result.provider, result.latency)
+"""
+
+from .client import MyLLM, FallbackResult
+from .config import Provider, PROVIDERS, PROVIDER_CHAIN, PROVIDER_MAP
+from .server import app
+from .tracker import Tracker, tracker
+
+__version__ = "0.1.0"
+__all__ = [
+    "MyLLM",
+    "FallbackResult",
+    "Provider",
+    "PROVIDERS",
+    "PROVIDER_CHAIN",
+    "PROVIDER_MAP",
+    "Tracker",
+    "tracker",
+    "app",
+]

myllm/__main__.py

@@ -0,0 +1,3 @@
+from .server import main
+
+main()

myllm/client.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import json
+import urllib.request
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class FallbackResult:
+    """Result from a chat completion with provider metadata.
+
+    Attributes:
+        content: The generated text response.
+        provider: Name of the provider that fulfilled the request.
+        model: Model identifier used.
+        latency: Time taken in seconds.
+        attempts: List of failed attempts before success.
+    """
+
+    content: str
+    provider: str
+    model: str
+    latency: float
+    attempts: list[dict[str, Any]] = field(default_factory=list)
+
+
+def _call_server(server_url: str, params: dict[str, Any]) -> FallbackResult:
+    """Send a chat request to the myllm server.
+
+    Args:
+        server_url: Base URL of the myllm server.
+        params: Request parameters (messages, model, temperature, etc.).
+
+    Returns:
+        FallbackResult with generated content and metadata.
+
+    Raises:
+        RuntimeError: If the server returns an error or is unreachable.
+    """
+    url = f"{server_url.rstrip('/')}/chat/completions"
+    body = json.dumps(params).encode()
+    req = urllib.request.Request(
+        url,
+        data=body,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=300) as resp:
+            data = json.loads(resp.read())
+    except Exception as exc:
+        raise RuntimeError(f"Server request failed: {exc}") from exc
+
+    if "error" in data:
+        raise RuntimeError(data["error"])
+
+    return FallbackResult(
+        content=data.get("content", ""),
+        provider=data.get("provider", ""),
+        model=data.get("model", ""),
+        latency=data.get("latency", 0),
+        attempts=data.get("attempts", []),
+    )
+
+
+class MyLLM:
+    """Client for the myllm multi-provider LLM server.
+
+    Connects to a running myllm server and provides a simple interface
+    for chat completions with automatic provider fallback.
+
+    Args:
+        timeout: Request timeout in seconds. Defaults to 30.0.
+        server_url: myllm server URL. Defaults to "http://localhost:10000".
+        max_wait: Maximum wait time for provider availability in seconds.
+            Defaults to 80.0.
+
+    Example:
+        >>> client = MyLLM(server_url="http://localhost:10000")
+        >>> result = client.chat([{"role": "user", "content": "Hello"}])
+        >>> print(result.content)
+        >>> print(result.provider, result.latency)
+    """
+
+    def __init__(
+        self,
+        *,
+        timeout: float = 30.0,
+        server_url: str = "http://localhost:10000",
+        max_wait: float = 80.0,
+    ) -> None:
+        self.timeout = timeout
+        self.server_url = server_url
+        self.max_wait = max_wait
+
+    def chat(
+        self,
+        messages: list[dict[str, Any]],
+        *,
+        model: str | None = None,
+        temperature: float | None = None,
+        max_tokens: int | None = None,
+        debug_provider: str | None = None,
+        max_wait: float | None = None,
+        **kwargs: Any,
+    ) -> FallbackResult:
+        """Send a chat completion request.
+
+        Args:
+            messages: List of message dicts with 'role' and 'content' keys.
+            model: Model identifier override. Uses provider default if None.
+            temperature: Sampling temperature (0.0-2.0).
+            max_tokens: Maximum tokens to generate.
+            debug_provider: Force a specific provider name (e.g. "groq").
+            max_wait: Override max wait time for this request.
+            **kwargs: Additional parameters passed to the server.
+
+        Returns:
+            FallbackResult with generated content and metadata.
+
+        Example:
+            >>> result = client.chat(
+            ...     [{"role": "user", "content": "Explain quantum computing"}],
+            ...     temperature=0.7,
+            ...     max_tokens=500,
+            ... )
+        """
+        params: dict[str, Any] = {"messages": messages, **kwargs}
+        if model is not None:
+            params["model"] = model
+        if temperature is not None:
+            params["temperature"] = temperature
+        if max_tokens is not None:
+            params["max_tokens"] = max_tokens
+        if debug_provider is not None:
+            params["debug_provider"] = debug_provider
+        params["max_wait"] = max_wait if max_wait is not None else self.max_wait
+        params.setdefault("timeout", self.timeout)
+        return _call_server(self.server_url, params)
+
+    def complete(self, prompt: str, **kwargs: Any) -> FallbackResult:
+        """Convenience method to send a single user prompt.
+
+        Args:
+            prompt: The user message text.
+            **kwargs: Additional arguments passed to chat().
+
+        Returns:
+            FallbackResult with generated content and metadata.
+
+        Example:
+            >>> result = client.complete("What is 2+2?")
+            >>> print(result.content)
+        """
+        return self.chat([{"role": "user", "content": prompt}], **kwargs)

myllm/config.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Provider:
+    """AI provider configuration.
+
+    Attributes:
+        name: Provider identifier (e.g. "groq", "cerebras").
+        base_url: API base URL.
+        api_key_env: Environment variable name for the API key.
+        model: Default model identifier.
+        priority: Lower number = higher priority.
+        rpm: Requests per minute limit (0 = unlimited).
+        rpd: Requests per day limit (0 = unlimited).
+    """
+
+    name: str
+    base_url: str
+    api_key_env: str
+    model: str
+    priority: int = 0
+    rpm: int = 0
+    rpd: int = 0
+
+
+PROVIDERS: list[Provider] = [
+    Provider(
+        name="cerebras",
+        base_url="https://api.cerebras.ai/v1",
+        api_key_env="CEREBRAS_API_KEY",
+        model="zai-glm-4.7",
+        priority=1,
+        rpm=5,
+    ),
+    Provider(
+        name="groq",
+        base_url="https://api.groq.com/openai/v1",
+        api_key_env="GROQ_API_KEY",
+        model="openai/gpt-oss-120b",
+        priority=2,
+        rpm=30,
+        rpd=1000,
+    ),
+    Provider(
+        name="openrouter",
+        base_url="https://openrouter.ai/api/v1",
+        api_key_env="OPENROUTER_API_KEY",
+        model="google/gemma-4-31b-it:free",
+        priority=3,
+        rpm=20,
+        rpd=1000,
+    ),
+    Provider(
+        name="nvidia_nim",
+        base_url="https://integrate.api.nvidia.com/v1",
+        api_key_env="NVIDIA_API_KEY",
+        model="openai/gpt-oss-120b",
+        priority=4,
+        rpm=40,
+    ),
+]
+
+PROVIDER_CHAIN: list[Provider] = sorted(PROVIDERS, key=lambda p: p.priority)
+PROVIDER_MAP: dict[str, Provider] = {p.name: p for p in PROVIDERS}

myllm/diagnose.py

@@ -0,0 +1,30 @@
+"""Diagnose: test each provider individually (1 request each)."""
+from __future__ import annotations
+
+import sys
+import time
+
+sys.stdout.reconfigure(line_buffering=True)
+
+from myllm.server import _route
+from myllm.config import PROVIDERS
+from myllm.tracker import tracker
+
+PROMPT = "Say hello in one sentence."
+
+for p in PROVIDERS:
+    print(f"\n--- {p.name} ({p.model}) ---")
+    t0 = time.monotonic()
+    result = _route(
+        messages=[{"role": "user", "content": PROMPT}],
+        debug_provider=p.name,
+        max_tokens=256,
+    )
+    lat = time.monotonic() - t0
+    if "error" in result:
+        print(f"  FAIL ({lat:.2f}s): {result['error'][:120]}")
+    else:
+        print(f"  OK   ({lat:.2f}s): {result['content'][:80]}")
+    print(f"  tracker: {tracker.snapshot()}")
+
+print("\nDone.")

myllm/lib.py

@@ -0,0 +1,10 @@
+"""Utility functions for myllm."""
+
+
+def main() -> None:
+    """Entry point for myllm CLI."""
+    print("Hello from myllm!")
+
+
+if __name__ == "__main__":
+    main()

myllm/plain_execute.py

@@ -0,0 +1,275 @@
+"""Benchmark: 60 parallel requests, 120s hard timeout."""
+from __future__ import annotations
+
+import statistics
+import sys
+import threading
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+sys.stdout.reconfigure(line_buffering=True)
+
+from myllm.server import _route
+
+TOTAL = 60
+TIMEOUT = 120.0
+
+PROMPTS = [
+    "대학원 레벨로 3차원 토러스의 가우스 곡률을 섹셔널 커버쳐 공식으로 유도하고, 오일러 특성 수를 계산해",
+    "리만 기하학에서 슈바르츠실드 계량의 크리스토펠 기호를 계산하고, 빛의 편향각을 뉴턴 근사로 유도해",
+    "양자역학에서 수소원자의 n=3, l=2, m=1 상태의 파동함수를 구하고, 각도분포함수를 그래프로 설명해",
+    "콕크로프트-월턴 가속기의 리플 전압을 Fourier 해석하고, 출력 에너지 스펙트럼의 반치폭을 유도해",
+    "열역학에서 반데르발스 기체의 보일 온도를 유도하고, 임계점에서의 압력-부피-온도 관계를 전개해",
+    "전자기학에서 전자기파의 스넬 법칙을 프레넬 방정식으로 유도하고, 브루스터 각에서의 반사율을 계산해",
+    "유체역학에서 나비에-스토크스 방정식을 무차원화하고, 레이놀즈 수에 따른 층류-난류 천이를 설명해",
+    "고체물리학에서 브릴루앙 영역의 페르미 표면을 자유전자 모델로 구하고, 밴드 구조를 설명해",
+    "미분기하학에서 리치 흐름 방정식을 2차원 구면에 대해 풀고, 곡률이 시간에 따라 변하는 과정을 전개해",
+    "응집물리학에서 BCS 이론으로 초전도체의 에너지 갭을 유도하고, 임계온도와의 관계를 설명해",
+    "양자장론에서 자유 스칼라장의 파동방정식을 그린 함수로 풀고, 전파자를 유도해",
+    "상대론적 역학에서 쌍생성 에너지를 계산하고, 전자-양전자 쌍의 운동량 보존을 전개해",
+    "천체물리학에서 토만-오펜하이머-볼코프 한계를 유도하고, 중성자별의 최대 질량을 추정해",
+    "통계역학에서 이상 보즈-아인슈타인 응축의 임계온도를 유도하고, 응축 비율을 온도 함수로 전개해",
+    "광학에서 파브리-페로 간섭계의 자유 스펙트럼 범위와 분해능을 유도하고, 피네스와의 관계를 설명해",
+    "입자물리학에서 쿼크 모형으로 중성자의 자기 모멘트를 계산하고, 양성자와의 비율을 유도해",
+    "편미분방정식에서 열방정식의 푸리에 해법을 유도하고, 그린 함수 방법으로 초기값 문제를 풀어",
+    "대수학에서 갈루아 이론으로 5차 방정식의 비가해성을 증명하고, 갈루아 군의 구조를 분석해",
+    "해석학에서 리만 제타 함수의 함수 방정식을 유도하고, 영점 분포와 소수 정리와의 관계를 설명해",
+    "위상수학에서 베티 수를 CW 복합체로 계산하고, 오일러 지표와의 관계를 전개해",
+    "확률론에서 중심극한정리를 특성함수 방법으로 증명하고, 수렴 속도를 베리-에센 정리로 추정해",
+    "수치해석에서 룽게-쿠타 현상의 원인을 분석하고, 체비셰프 점을 사용한 다항식 보간의 안정성을 증명해",
+    "제어이론에서 PID 제어기의 안정 영역을 루트 궤적법으로 구하고, 위상 여유와 이득 여유를 계산해",
+    "신호처리에서 이산 푸리에 변환의 원형 컨볼루션 정리를 증명하고, 빠른 푸리에 변환의 계산 복잡도를 분석해",
+    "정보이론에서 샤논 채널 용량定理을 증명하고, AWGN 채널에서의 최적 전달률을 유도해",
+    "기계학습에서 서포트 벡터 머신의 라그랑지 쌍대 문제를 유도하고, 커널 트릭의 수학적 근거를 설명해",
+    "최적화 이론에서 카루시-쿤타 조건을 유도하고, 등식 제약이 있는 라그랑지 승수법의 수렴성을 증명해",
+    "그래프 이론에서 맥스플로-민컷 정리를 증명하고, 에드몬드-카프 알고리즘의 시간복잡도를 분석해",
+    "암호학에서 RSA의 안전성을 정수 분해 문제로 환원하고, 밀러-라빈 소수 판별법의 오류 확률을 계산해",
+    "컴파일러 이론에서 LR 파서의 충돌 검출 알고리즘을 설계하고, LALR 테이블 생성 과정을 전개해",
+    "분산시스템에서 CAP 정리를 증명하고, Paxos 합의 알고리즘의 안전성을 형식적으로 검증해",
+    "운영체제에서 페이지 교체 알고리즘의 경쟁 비율을 분석하고, LRU의 최악 경우를 예시로 보여줘",
+    "데이터베이스에서 B+ 트리의 삽입/삭제 알고리즘을 분석하고, 높이와 디스크 접근의 관계를 유도해",
+    "네트워크에서 TCP의 혼잡 제어 알고리즘을 수학적으로 모델링하고, 공평성을 분석해",
+    "양자컴퓨팅에서 쇼어 알고리즘의 양자 푸리에 변환을 전개하고, 주기 찾기의 정확도를 분석해",
+    "블록체인에서 비잔틴 장군 문제의 합의 조건을 형식화하고, PBFT의 안전성을 증명해",
+    "로봇공학에서 역기구학의 해를 뉴턴-랩슨 방법으로 구하고, 특이점 분석을 전개해",
+    "영상처리에서 허프 변환의 수학적 원리를 유도하고, 원 검출의 정확도를 분석해",
+    "음성신호처리에서 LPC 분석을 유도하고, 음성 합성 필터의 안정성을 증명해",
+    "계산복잡도에서 P=NP 문제의 NP-완전성 증명 예시로 SAT 문제를 사용하고, 쿡 정리를 전개해",
+    "형식언어 이론에서 푸싱다운 오토마타와 CFG의 동치성을 증명하고, 결정화 가능 조건을 분석해",
+    "추상대수학에서 유한체 GF(2^8)의 곱셈 구조를 분석하고, AES에서 사용하는 기약 다항식을 유도해",
+    "대수기하학에서 베주 정리를 증명하고, 힐베르트 영점定理의 기하학적 의미를 설명해",
+    "콕세터 군의 반사 표현을 유도하고, 와일 군의 분류를 전개해",
+    "호모토피 이론에서 기본군을 CW 복합체로 계산하고, 판-트리콥토의 기본군을 구해",
+    "K-이론에서 벡터 다발의 휘트니 합과 텐서곱을 정의하고, 복소 투영 공간의 K-군을 계산해",
+    "호모로지 대수학에서 Ext와 Tor 함자를 유도하고, 유니버설 계수 정리를 증명해",
+    "모형론에서 괘델 완전성定理을 증명하고, 로벤하임-스콜렘定理의 철학적 함의를 설명해",
+    "집합론에서 선택 공리와 초른 보조정리의 동치성을 증명하고, 바나흐-타르스키 역설을 전개해",
+    "수리논리에서 Church-Turing thesis를 형식화하고, 정지 문제의 비결정성을 증명해",
+]
+
+PROMPT = "대학원 레벨로 3차원 토러스 섹셔널 커버쳐 사용해서 오일러 넘버 산출 전개 과정 전개해"
+
+
+@dataclass
+class Req:
+    seq: int
+    ok: bool
+    latency: float
+    provider: str = ""
+    model: str = ""
+    error: str = ""
+    chars: int = 0
+
+
+@dataclass
+class Stats:
+    results: list[Req] = field(default_factory=list)
+    lock: threading.Lock = field(default_factory=threading.Lock)
+    done: int = 0
+
+    def add(self, r: Req) -> None:
+        with self.lock:
+            self.results.append(r)
+            self.done += 1
+
+    @property
+    def ok(self) -> list[Req]:
+        with self.lock:
+            return [r for r in self.results if r.ok]
+
+    @property
+    def fail(self) -> list[Req]:
+        with self.lock:
+            return [r for r in self.results if not r.ok]
+
+
+_stop = threading.Event()
+
+
+def _send_one(seq: int, stats: Stats) -> None:
+    if _stop.is_set():
+        return
+    prompt = PROMPTS[seq % len(PROMPTS)]
+    t0 = time.monotonic()
+    try:
+        result = _route(
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=60000,
+            max_wait=80.0,
+        )
+        lat = time.monotonic() - t0
+        if "error" in result:
+            req = Req(seq=seq, ok=False, latency=lat,
+                      provider=result.get("provider", ""),
+                      model=result.get("model", ""),
+                      error=result["error"][:120])
+        else:
+            req = Req(seq=seq, ok=True, latency=lat,
+                      provider=result["provider"],
+                      model=result["model"],
+                      chars=len(result.get("content", "")))
+    except Exception as exc:
+        lat = time.monotonic() - t0
+        req = Req(seq=seq, ok=False, latency=lat, error=str(exc)[:120])
+
+    stats.add(req)
+    mark = "OK" if req.ok else "FAIL"
+    print(f"[{req.seq:02d}] {mark} {req.latency:5.2f}s  "
+          f"provider={req.provider}/{req.model}  "
+          f"ok={len(stats.ok)}/{stats.done}", flush=True)
+
+
+def run() -> tuple[Stats, float]:
+    stats = Stats()
+    wall_start = time.monotonic()
+
+    threads: list[threading.Thread] = []
+    for i in range(TOTAL):
+        t = threading.Thread(target=_send_one, args=(i, stats), daemon=True)
+        t.start()
+        threads.append(t)
+
+    deadline = wall_start + TIMEOUT
+    while time.monotonic() < deadline:
+        if stats.done >= TOTAL:
+            break
+        time.sleep(0.2)
+
+    _stop.set()
+    wall_time = time.monotonic() - wall_start
+    print(f"\n--- wall {wall_time:.1f}s, completed {stats.done}/{TOTAL} ---", flush=True)
+    return stats, wall_time
+
+
+def report(stats: Stats, wall_time: float) -> None:
+    ok = stats.ok
+    fail = stats.fail
+    lats = [r.latency for r in ok]
+    rpm = len(ok) / (wall_time / 60) if wall_time > 0 else 0
+
+    print("\n" + "=" * 60)
+    print("BENCHMARK REPORT")
+    print("=" * 60)
+    print(f"Sent:        {stats.done}")
+    print(f"OK:          {len(ok)}")
+    print(f"Fail:        {len(fail)}")
+    print(f"Wall time:   {wall_time:.1f}s")
+    print(f"Throughput:  {rpm:.1f} ok/min")
+
+    if lats:
+        sl = sorted(lats)
+        p50 = sl[len(sl) // 2]
+        p95 = sl[min(int(len(sl) * 0.95), len(sl) - 1)]
+        print(f"Latency min: {min(lats):.2f}s")
+        print(f"Latency max: {max(lats):.2f}s")
+        print(f"Latency avg: {statistics.mean(lats):.2f}s")
+        print(f"Latency p50: {p50:.2f}s")
+        print(f"Latency p95: {p95:.2f}s")
+        if len(lats) >= 2:
+            print(f"Latency std: {statistics.stdev(lats):.2f}s")
+
+    by_prov: dict[str, dict[str, int]] = {}
+    for r in stats.results:
+        k = f"{r.provider}/{r.model}" if r.provider else "unknown"
+        by_prov.setdefault(k, {"ok": 0, "fail": 0})
+        if r.ok:
+            by_prov[k]["ok"] += 1
+        else:
+            by_prov[k]["fail"] += 1
+
+    if by_prov:
+        print("\nPer-provider:")
+        for k, v in sorted(by_prov.items()):
+            print(f"  {k}: ok={v['ok']}  fail={v['fail']}")
+
+    if fail:
+        print("\nErrors (sample):")
+        seen: set[str] = set()
+        for r in fail:
+            if r.error not in seen:
+                seen.add(r.error)
+                print(f"  - {r.error[:100]}")
+
+    md = [
+        "# Benchmark Report",
+        "",
+        f"**Date**: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}  ",
+        f"**Wall time**: {wall_time:.1f}s  ",
+        f"**Timeout**: {TIMEOUT}s  ",
+        f"**Mode**: parallel (threads)  ",
+        "",
+        "## Summary",
+        "",
+        "| Metric | Value |",
+        "|--------|-------|",
+        f"| Sent | {stats.done} |",
+        f"| OK | {len(ok)} |",
+        f"| Fail | {len(fail)} |",
+        f"| Throughput (ok/min) | **{rpm:.1f}** |",
+        "",
+    ]
+    if lats:
+        md += [
+            "## Latency",
+            "",
+            "| Metric | Value |",
+            "|--------|-------|",
+            f"| Min | {min(lats):.2f}s |",
+            f"| Max | {max(lats):.2f}s |",
+            f"| Mean | {statistics.mean(lats):.2f}s |",
+            f"| p50 | {p50:.2f}s |",
+            f"| p95 | {p95:.2f}s |",
+            "",
+        ]
+    if by_prov:
+        md += [
+            "## Per-provider",
+            "",
+            "| Provider/Model | OK | Fail |",
+            "|----------------|----|------|",
+        ]
+        for k, v in sorted(by_prov.items()):
+            md.append(f"| {k} | {v['ok']} | {v['fail']} |")
+        md.append("")
+
+    if fail:
+        md += ["## Errors", ""]
+        seen: set[str] = set()
+        for r in fail:
+            if r.error not in seen:
+                seen.add(r.error)
+                md.append(f"- `{r.error[:100]}`")
+        md.append("")
+
+    out = Path(__file__).parent.parent / "tests" / "benchmark_report.md"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text("\n".join(md), encoding="utf-8")
+    print(f"\nReport: {out}")
+
+
+if __name__ == "__main__":
+    stats, wall_time = run()
+    report(stats, wall_time)
+    sys.exit(0 if stats.ok else 1)

myllm/server.py

@@ -0,0 +1,253 @@
+"""FastAPI server — provider routing, fallback, RPM/RPD tracking.
+
+Provides a /chat/completions endpoint that routes requests through
+multiple AI providers with automatic fallback, rate limiting, and
+cooldown tracking.
+
+Endpoints:
+    POST /chat/completions: Chat completion with provider fallback.
+    GET /health: Health check.
+    GET /tracker: Provider rate limit and cooldown status.
+"""
+from __future__ import annotations
+
+import os
+import threading
+import time
+from typing import Any
+
+import uvicorn
+from fastapi import FastAPI
+from pydantic import BaseModel
+from dotenv import load_dotenv
+from pathlib import Path
+
+load_dotenv(Path(__file__).parent.parent / ".env")
+
+from openai import OpenAI, APIStatusError, APITimeoutError, APIConnectionError
+
+from .config import PROVIDER_CHAIN, PROVIDER_MAP, Provider
+from .tracker import tracker
+
+app = FastAPI(title="myllm server")
+_claim_lock = threading.Lock()
+
+
+def _make_client(provider: Provider) -> OpenAI | None:
+    """Create an OpenAI client for the given provider.
+
+    Args:
+        provider: Provider configuration with base_url and api_key_env.
+
+    Returns:
+        OpenAI client instance, or None if API key is not set.
+    """
+    api_key = os.environ.get(provider.api_key_env, "")
+    if not api_key:
+        return None
+    return OpenAI(base_url=provider.base_url, api_key=api_key, timeout=30.0)
+
+
+RETRY_INTERVAL = 20.0
+DEFAULT_MAX_WAIT = 80.0
+
+
+def _claim_provider(model: str | None = None) -> Provider | None:
+    """Claim an available provider from the rate-limited chain.
+
+    Args:
+        model: Unused, reserved for future model-based routing.
+
+    Returns:
+        Available Provider, or None if all are at capacity.
+    """
+    with _claim_lock:
+        for p in PROVIDER_CHAIN:
+            if tracker.is_available(p.name, p.rpm, p.rpd):
+                tracker.record_claim(p.name)
+                return p
+    return None
+
+
+def _try_api_call(
+    provider: Provider,
+    messages: list[dict[str, Any]],
+    *,
+    model: str | None = None,
+    temperature: float | None = None,
+    max_tokens: int | None = None,
+    timeout: float | None = None,
+    **kwargs: Any,
+) -> dict[str, Any]:
+    client = _make_client(provider)
+    if client is None:
+        raise RuntimeError(f"No API key for {provider.name}")
+
+    m = model or provider.model
+    params: dict[str, Any] = {
+        "model": m,
+        "messages": messages,
+        **kwargs,
+    }
+    if temperature is not None:
+        params["temperature"] = temperature
+    if max_tokens is not None:
+        params["max_tokens"] = max_tokens
+    params.setdefault("timeout", timeout or 30.0)
+
+    t0 = time.monotonic()
+    resp = client.chat.completions.create(**params)
+    elapsed = time.monotonic() - t0
+
+    tracker.record_success(provider.name)
+    msg = resp.choices[0].message
+    content = msg.content or ""
+    if not content:
+        reasoning = getattr(msg, "reasoning", None) or getattr(msg, "reasoning_content", None)
+        if reasoning:
+            content = reasoning
+
+    return {
+        "content": content,
+        "provider": provider.name,
+        "model": m,
+        "latency": elapsed,
+    }
+
+
+def _route(
+    messages: list[dict[str, Any]],
+    *,
+    model: str | None = None,
+    temperature: float | None = None,
+    max_tokens: int | None = None,
+    timeout: float | None = None,
+    debug_provider: str | None = None,
+    max_wait: float = DEFAULT_MAX_WAIT,
+    **kwargs: Any,
+) -> dict[str, Any]:
+    errors: list[dict[str, Any]] = []
+
+    if debug_provider:
+        provider = PROVIDER_MAP.get(debug_provider)
+        if provider is None:
+            return {"error": f"Unknown provider: {debug_provider}"}
+        try:
+            return _try_api_call(
+                provider, messages, model=model, temperature=temperature,
+                max_tokens=max_tokens, timeout=timeout, **kwargs,
+            )
+        except Exception as exc:
+            return {"error": str(exc)[:200], "attempts": []}
+
+    deadline = time.monotonic() + max_wait
+    while True:
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            break
+
+        provider = _claim_provider()
+        if provider is None:
+            wait = min(RETRY_INTERVAL, remaining)
+            time.sleep(wait)
+            continue
+
+        try:
+            result = _try_api_call(
+                provider, messages, model=model, temperature=temperature,
+                max_tokens=max_tokens, timeout=timeout, **kwargs,
+            )
+            result["attempts"] = errors
+            return result
+        except (APIStatusError, APITimeoutError, APIConnectionError, Exception) as exc:
+            tracker.record_failure(provider.name)
+            errors.append({
+                "provider": provider.name,
+                "model": model or provider.model,
+                "error": str(exc)[:200],
+            })
+            continue
+
+    return {"error": "All providers failed", "attempts": errors}
+
+
+class ChatRequest(BaseModel):
+    """Request schema for /chat/completions endpoint.
+
+    Attributes:
+        messages: List of chat messages.
+        model: Model identifier override.
+        temperature: Sampling temperature (0.0-2.0).
+        max_tokens: Maximum tokens to generate.
+        timeout: Per-request timeout in seconds.
+        debug_provider: Force a specific provider name.
+        max_wait: Max wait time for provider availability.
+    """
+
+    messages: list[dict[str, Any]]
+    model: str | None = None
+    temperature: float | None = None
+    max_tokens: int | None = None
+    timeout: float | None = None
+    debug_provider: str | None = None
+    max_wait: float = DEFAULT_MAX_WAIT
+
+
+@app.post("/chat/completions")
+def chat_completions(req: ChatRequest) -> dict[str, Any]:
+    """Chat completion endpoint with multi-provider fallback.
+
+    Routes the request through available providers in priority order.
+    Automatically falls back to the next provider on failure.
+
+    Args:
+        req: ChatRequest with messages and optional parameters.
+
+    Returns:
+        Dict with content, provider, model, latency, and attempts.
+    """
+    return _route(
+        messages=req.messages,
+        model=req.model,
+        temperature=req.temperature,
+        max_tokens=req.max_tokens,
+        timeout=req.timeout,
+        debug_provider=req.debug_provider,
+        max_wait=req.max_wait,
+    )
+
+
+@app.get("/health")
+def health() -> dict[str, Any]:
+    """Health check endpoint.
+
+    Returns:
+        Dict with status "ok".
+    """
+    return {"status": "ok"}
+
+
+@app.get("/tracker")
+def tracker_status() -> dict[str, Any]:
+    """Provider rate limit and cooldown status.
+
+    Returns:
+        Dict mapping provider names to their current RPM, RPD,
+        fail count, and cooldown status.
+    """
+    return tracker.snapshot()
+
+
+def main() -> None:
+    import argparse
+
+    parser = argparse.ArgumentParser(description="myllm API server")
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=10000)
+    args = parser.parse_args()
+
+    uvicorn.run(app, host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+    main()

myllm/test.py

@@ -0,0 +1,227 @@
+"""Benchmark: 60 requests in parallel, 1-minute wall-clock timeout.
+
+Usage:
+    uv run python -m myllm.test           # via server (RPM-controlled)
+    uv run python -m myllm.test --direct  # direct provider calls
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import statistics
+import sys
+import threading
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv(Path(__file__).parent.parent / ".env")
+
+sys.stdout.reconfigure(line_buffering=True)
+
+from myllm import MyLLM
+
+TOTAL = 60
+TIMEOUT = 60.0
+PER_REQ_TIMEOUT = 30.0
+DEFAULT_SERVER = "http://localhost:10000"
+
+PROMPT = (
+    "What is the Gaussian curvature of a torus with major radius R=3 and minor radius r=1 "
+    "at the point where theta=pi/4? Show the formula and numerical result in 3 lines."
+)
+
+
+@dataclass
+class Req:
+    seq: int
+    ok: bool
+    latency: float
+    provider: str = ""
+    model: str = ""
+    error: str = ""
+    chars: int = 0
+
+
+@dataclass
+class Stats:
+    results: list[Req] = field(default_factory=list)
+    lock: threading.Lock = field(default_factory=threading.Lock)
+    done: int = 0
+
+    def add(self, r: Req) -> None:
+        with self.lock:
+            self.results.append(r)
+            self.done += 1
+
+    @property
+    def ok(self) -> list[Req]:
+        with self.lock:
+            return [r for r in self.results if r.ok]
+
+    @property
+    def fail(self) -> list[Req]:
+        with self.lock:
+            return [r for r in self.results if not r.ok]
+
+
+_stop = threading.Event()
+
+
+def _send_one(llm: MyLLM, seq: int, stats: Stats) -> None:
+    if _stop.is_set():
+        return
+    t0 = time.monotonic()
+    try:
+        result = llm.complete(PROMPT)
+        lat = time.monotonic() - t0
+        req = Req(seq=seq, ok=True, latency=lat,
+                  provider=result.provider, model=result.model,
+                  chars=len(result.content))
+    except Exception as exc:
+        lat = time.monotonic() - t0
+        req = Req(seq=seq, ok=False, latency=lat, error=str(exc)[:120])
+
+    stats.add(req)
+    mark = "OK" if req.ok else "FAIL"
+    print(f"[{req.seq:02d}] {mark} {req.latency:5.2f}s  "
+          f"provider={req.provider}/{req.model}  "
+          f"ok={len(stats.ok)}/{stats.done}", flush=True)
+
+
+def run(server_url: str | None = None) -> tuple[Stats, float]:
+    llm = MyLLM(timeout=PER_REQ_TIMEOUT, server_url=server_url)
+    stats = Stats()
+    wall_start = time.monotonic()
+
+    threads: list[threading.Thread] = []
+    for i in range(TOTAL):
+        t = threading.Thread(target=_send_one, args=(llm, i, stats), daemon=True)
+        t.start()
+        threads.append(t)
+
+    deadline = wall_start + TIMEOUT
+    while time.monotonic() < deadline:
+        if stats.done >= TOTAL:
+            break
+        time.sleep(0.2)
+
+    _stop.set()
+    wall_time = time.monotonic() - wall_start
+    print(f"\n--- wall {wall_time:.1f}s, completed {stats.done}/{TOTAL} ---", flush=True)
+    return stats, wall_time
+
+
+def report(stats: Stats, wall_time: float) -> None:
+    ok = stats.ok
+    fail = stats.fail
+    lats = [r.latency for r in ok]
+    rpm = len(ok) / (wall_time / 60) if wall_time > 0 else 0
+
+    print("\n" + "=" * 60)
+    print("BENCHMARK REPORT")
+    print("=" * 60)
+    print(f"Sent:        {stats.done}")
+    print(f"OK:          {len(ok)}")
+    print(f"Fail:        {len(fail)}")
+    print(f"Wall time:   {wall_time:.1f}s")
+    print(f"Throughput:  {rpm:.1f} ok/min")
+
+    if lats:
+        sl = sorted(lats)
+        p50 = sl[len(sl) // 2]
+        p95 = sl[min(int(len(sl) * 0.95), len(sl) - 1)]
+        print(f"Latency min: {min(lats):.2f}s")
+        print(f"Latency max: {max(lats):.2f}s")
+        print(f"Latency avg: {statistics.mean(lats):.2f}s")
+        print(f"Latency p50: {p50:.2f}s")
+        print(f"Latency p95: {p95:.2f}s")
+        if len(lats) >= 2:
+            print(f"Latency std: {statistics.stdev(lats):.2f}s")
+
+    by_prov: dict[str, dict[str, int]] = {}
+    for r in stats.results:
+        k = f"{r.provider}/{r.model}" if r.provider else "unknown"
+        by_prov.setdefault(k, {"ok": 0, "fail": 0})
+        if r.ok:
+            by_prov[k]["ok"] += 1
+        else:
+            by_prov[k]["fail"] += 1
+
+    if by_prov:
+        print("\nPer-provider:")
+        for k, v in sorted(by_prov.items()):
+            print(f"  {k}: ok={v['ok']}  fail={v['fail']}")
+
+    if fail:
+        print("\nErrors (sample):")
+        seen: set[str] = set()
+        for r in fail:
+            if r.error not in seen:
+                seen.add(r.error)
+                print(f"  - {r.error[:100]}")
+
+    md = [
+        "# Benchmark Report",
+        "",
+        f"**Date**: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}  ",
+        f"**Wall time**: {wall_time:.1f}s  ",
+        f"**Per-request timeout**: {PER_REQ_TIMEOUT}s  ",
+        f"**Mode**: parallel (threads)  ",
+        "",
+        "## Summary",
+        "",
+        "| Metric | Value |",
+        "|--------|-------|",
+        f"| Sent | {stats.done} |",
+        f"| OK | {len(ok)} |",
+        f"| Fail | {len(fail)} |",
+        f"| Throughput (ok/min) | **{rpm:.1f}** |",
+        "",
+    ]
+    if lats:
+        md += [
+            "## Latency",
+            "",
+            "| Metric | Value |",
+            "|--------|-------|",
+            f"| Min | {min(lats):.2f}s |",
+            f"| Max | {max(lats):.2f}s |",
+            f"| Mean | {statistics.mean(lats):.2f}s |",
+            f"| p50 | {p50:.2f}s |",
+            f"| p95 | {p95:.2f}s |",
+            "",
+        ]
+    if by_prov:
+        md += [
+            "## Per-provider",
+            "",
+            "| Provider/Model | OK | Fail |",
+            "|----------------|----|------|",
+        ]
+        for k, v in sorted(by_prov.items()):
+            md.append(f"| {k} | {v['ok']} | {v['fail']} |")
+        md.append("")
+
+    out = Path(__file__).parent.parent / "tests" / "benchmark_report.md"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text("\n".join(md), encoding="utf-8")
+    print(f"\nReport: {out}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--direct", action="store_true", help="Call providers directly (no server)")
+    parser.add_argument("--server", default=DEFAULT_SERVER, help="Server URL")
+    args = parser.parse_args()
+
+    server_url = None if args.direct else args.server
+    mode = "direct" if args.direct else f"server ({server_url})"
+    print(f"Mode: {mode}", flush=True)
+
+    stats, wall_time = run(server_url)
+    report(stats, wall_time)
+    os._exit(0 if stats.ok else 1)

myllm/tracker.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import time
+from collections import defaultdict, deque
+from dataclasses import dataclass, field
+
+
+@dataclass
+class _ProviderState:
+    """Internal state for tracking a single provider's rate limits."""
+
+    rpm_window: deque[float] = field(default_factory=lambda: deque())
+    rpd_window: deque[float] = field(default_factory=lambda: deque())
+    fail_count: int = 0
+    cooldown_until: float = 0.0
+
+
+class Tracker:
+    """Rate limit and cooldown tracker for AI providers.
+
+    Tracks requests per minute (RPM) and requests per day (RPD) for
+    each provider. Implements exponential backoff on failures.
+
+    Example:
+        >>> tracker = Tracker()
+        >>> tracker.is_available("groq", rpm=30, rpd=1000)
+        True
+        >>> tracker.record_claim("groq")
+        >>> tracker.record_success("groq")
+        >>> tracker.snapshot()
+        {'groq': {'rpm': 1, 'rpd': 1, 'fail_count': 0, 'cooldown_left': 0}}
+    """
+
+    def __init__(self) -> None:
+        self._states: dict[str, _ProviderState] = defaultdict(_ProviderState)
+
+    def _prune(self, state: _ProviderState, now: float) -> None:
+        while state.rpm_window and now - state.rpm_window[0] > 60:
+            state.rpm_window.popleft()
+        while state.rpd_window and now - state.rpd_window[0] > 86400:
+            state.rpd_window.popleft()
+
+    def _is_cooling_down(self, state: _ProviderState, now: float) -> bool:
+        return now < state.cooldown_until
+
+    def _all_others_cooling(self, provider: str, now: float) -> bool:
+        for name, state in self._states.items():
+            if name == provider:
+                continue
+            if not self._is_cooling_down(state, now):
+                return False
+        return bool(self._states)
+
+    def is_available(self, provider: str, rpm: int = 0, rpd: int = 0) -> bool:
+        """Check if a provider is available for a new request.
+
+        Args:
+            provider: Provider name.
+            rpm: Requests per minute limit.
+            rpd: Requests per day limit.
+
+        Returns:
+            True if the provider can accept a new request.
+        """
+        state = self._states[provider]
+        now = time.monotonic()
+
+        # nvidia_nim: always available when all others are cooling down
+        if provider == "nvidia_nim" and self._all_others_cooling(provider, now):
+            return True
+
+        if self._is_cooling_down(state, now):
+            return False
+
+        if rpm <= 0 and rpd <= 0:
+            return True
+
+        self._prune(state, now)
+        if rpm > 0 and len(state.rpm_window) >= rpm:
+            return False
+        if rpd > 0 and len(state.rpd_window) >= rpd:
+            return False
+        return True
+
+    def record_claim(self, provider: str) -> None:
+        """Record a request claim for rate limiting.
+
+        Args:
+            provider: Provider name.
+        """
+        now = time.monotonic()
+        state = self._states[provider]
+        state.rpm_window.append(now)
+        state.rpd_window.append(now)
+
+    def record_success(self, provider: str) -> None:
+        """Record a successful request, resetting failure count.
+
+        Args:
+            provider: Provider name.
+        """
+        state = self._states[provider]
+        state.fail_count = 0
+        state.cooldown_until = 0.0
+
+    def record_failure(self, provider: str) -> None:
+        """Record a failed request with exponential backoff.
+
+        Args:
+            provider: Provider name.
+        """
+        state = self._states[provider]
+        state.fail_count += 1
+        delay = min(120, 2 ** (state.fail_count - 1))
+        state.cooldown_until = time.monotonic() + delay
+
+    def snapshot(self) -> dict[str, dict]:
+        """Get current status of all tracked providers.
+
+        Returns:
+            Dict mapping provider names to their status including
+            RPM, RPD, fail count, and cooldown remaining.
+        """
+        now = time.monotonic()
+        out: dict[str, dict] = {}
+        for name, state in self._states.items():
+            self._prune(state, now)
+            cooling = self._is_cooling_down(state, now)
+            out[name] = {
+                "rpm": len(state.rpm_window),
+                "rpd": len(state.rpd_window),
+                "fail_count": state.fail_count,
+                "cooldown_left": max(0, state.cooldown_until - now) if cooling else 0,
+            }
+        return out
+
+
+tracker = Tracker()

simplyllm-0.1.0.dist-info/METADATA

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: simplyllm
+Version: 0.1.0
+Summary: Free AI LLM client with multi-provider fallback and rate limiting
+Project-URL: Repository, https://github.com/freeai/myllm
+License-Expression: MIT
+Requires-Python: >=3.9
+Requires-Dist: fastapi>=0.100.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: uvicorn>=0.20.0

simplyllm-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+myllm/__init__.py,sha256=p1RgiyY6bm5wO0KVaKaMUylYF5xhbOAJAbxweMfayfY,815
+myllm/__main__.py,sha256=3dYKHfmWsrdExFlTFlcR5a_icR9fAkn06Yh14TQkEd8,33
+myllm/client.py,sha256=Vx7k71qdbzVVhb2J2d65-tdC0nVqdARC1Jnr09VEaEU,5077
+myllm/config.py,sha256=BsK2_H21xK3i-YjozVdjTT0XDn9L-cWzgm3IJ7WL8pc,1701
+myllm/diagnose.py,sha256=haVha9EP91lzJrkQoZI0b2p8N8TR_ut_LDb63vb-4Sg,800
+myllm/lib.py,sha256=67pEgWqYpuFov_XT4rmziugn8sK4e-EsAwPe8Fdlxcs,165
+myllm/plain_execute.py,sha256=hqWQdieLkJfxrCK0R4-tCznq4w8Lc6wCTwSuf4_NcW4,12966
+myllm/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+myllm/server.py,sha256=22Uzaq8eOkawiSMlVbNl54ac5K0MRVHjaiu-faXQW2A,7152
+myllm/test.py,sha256=uL5AsF0rDbY-1KW8VzvqeohcaGk94NVgzQfJqIgJJPk,6637
+myllm/tracker.py,sha256=wyn-rMClHRC0KgJPLOa-RxLtt6_4LkFA4OJRHgehXZU,4445
+simplyllm-0.1.0.dist-info/METADATA,sha256=BG66rB4QH-wI16bGMssBA0VUtWxe5SJa9q1g5L3VrDY,360
+simplyllm-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+simplyllm-0.1.0.dist-info/entry_points.txt,sha256=qbDfb0GPF43wm5ioGV-wTRsUyYzG5iqYt13e2uKjprQ,44
+simplyllm-0.1.0.dist-info/RECORD,,

simplyllm-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

simplyllm-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+myllm = myllm.server:main