llm-cascade 0.1.0__py3-none-any.whl

llm_cascade/__init__.py

@@ -0,0 +1,26 @@
+"""llm-cascade: free-first LLM routing for Python.
+
+Routes each task through a cost ladder -- local Ollama ($0) -> OpenRouter free
+models ($0) -> paid Anthropic API -- with automatic fallback and a savings
+ledger. Zero required dependencies; install the `anthropic` extra to unlock
+the paid tiers.
+
+    from llm_cascade import route
+    result = route("summarize", "Long text here...")
+    print(result.text, result.tier, result.model, result.cost_usd)
+"""
+from __future__ import annotations
+
+from .router import Completion, complete, complete_json, pick, route, stats
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "route",
+    "complete",
+    "complete_json",
+    "pick",
+    "stats",
+    "Completion",
+    "__version__",
+]

llm_cascade/__main__.py

@@ -0,0 +1,7 @@
+"""Enables `python -m llm_cascade` as an alias for the `llm-cascade` console script."""
+from __future__ import annotations
+
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

llm_cascade/cli.py

@@ -0,0 +1,72 @@
+"""Command-line interface for llm-cascade.
+
+    llm-cascade "prompt here" --task summarize
+    llm-cascade --routes
+    llm-cascade --stats
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import config as _config
+from . import router
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="llm-cascade",
+        description="Free-first LLM routing: local Ollama -> OpenRouter free -> paid Anthropic.",
+    )
+    parser.add_argument("prompt", nargs="*", help="Prompt to complete")
+    parser.add_argument("--task", default="light",
+                         help="Task name or tier (trivial/local/cheap/light/heavy)")
+    parser.add_argument("--max-tokens", type=int, default=None)
+    parser.add_argument("--effort", default=None,
+                         choices=[None, "low", "medium", "high", "xhigh", "max"])
+    parser.add_argument("--routes", action="store_true",
+                         help="Print the tier ladder and task routing table")
+    parser.add_argument("--stats", action="store_true",
+                         help="Print cumulative cost and savings")
+    parser.add_argument("--json", action="store_true",
+                         help="Print machine-readable JSON instead of plain text")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    if args.routes:
+        if args.json:
+            print(json.dumps(_config.load_config()["task_routes"], indent=2, ensure_ascii=False))
+        else:
+            print(router.describe_routes())
+        return 0
+
+    if args.stats:
+        print(json.dumps(router.stats(), indent=2, ensure_ascii=False))
+        return 0
+
+    prompt = " ".join(args.prompt)
+    if not prompt:
+        print("error: no prompt given (pass a prompt, or use --routes / --stats)",
+              file=sys.stderr)
+        return 2
+
+    result = router.route(args.task, prompt, max_tokens=args.max_tokens, effort=args.effort)
+    if args.json:
+        print(json.dumps({
+            "text": result.text,
+            "tier": result.tier,
+            "model": result.model,
+            "cost_usd": result.cost_usd,
+        }, indent=2, ensure_ascii=False))
+    else:
+        print(f"[task={args.task} -> tier={result.tier} -> {result.model}]\n")
+        print(result.text)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

llm_cascade/config.py

@@ -0,0 +1,123 @@
+"""Configuration loading for llm-cascade.
+
+Precedence (lowest to highest):
+
+1. Built-in defaults (this module).
+2. An optional TOML file -- ``~/.llm_cascade/routes.toml`` by default, or the
+   path given in the ``LLM_CASCADE_CONFIG`` environment variable.
+3. Environment variables for the handful of settings that make sense as
+   env vars (API keys, HTTP headers, the log path, the free-cloud opt-in and
+   kill-switch). These are read directly by the modules that use them --
+   see ``router.py`` and ``providers/openrouter_provider.py`` -- rather than
+   being folded into the dict returned here.
+
+Call :func:`load_config` to get the merged config. It re-reads the file (if
+any) on every call, which keeps tests simple (no caching to reset) at a
+negligible cost for a CLI/library that isn't called thousands of times a
+second.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+try:
+    import tomllib  # Python >= 3.11
+except ModuleNotFoundError:  # pragma: no cover - exercised only on Python 3.10
+    import tomli as tomllib  # type: ignore[no-redef]
+
+DEFAULT_CONFIG_PATH = Path.home() / ".llm_cascade" / "routes.toml"
+DEFAULT_LOG_PATH = Path.home() / ".llm_cascade" / "log.jsonl"
+
+# The routing policy: which tier handles each task name. Anything not listed
+# here falls back to `default_task_tier`. This is the one table most users
+# will want to edit -- add task names via the TOML file's [task_routes].
+DEFAULT_TASK_ROUTES: dict[str, str] = {
+    "classify": "local",
+    "extract": "local",
+    "summarize": "local",
+    "translate": "local",
+    "rewrite": "light",
+    "code": "heavy",
+    "review": "heavy",
+    "plan": "heavy",
+    "creative": "heavy",
+    "chat": "local",
+}
+
+DEFAULTS: dict[str, Any] = {
+    "models": {
+        "opus": "claude-opus-4-8",
+        "sonnet": "claude-sonnet-5",
+        "haiku": "claude-haiku-4-5",
+    },
+    # USD per 1M tokens: [input, output]. Used only for the numbers shown by
+    # `llm-cascade --stats`; it never affects what a provider actually bills.
+    "pricing": {
+        "claude-opus-4-8": [5.0, 25.0],
+        "claude-sonnet-5": [2.0, 10.0],
+        "claude-haiku-4-5": [1.0, 5.0],
+    },
+    "task_routes": dict(DEFAULT_TASK_ROUTES),
+    "default_task_tier": "light",
+    # OpenRouter ":free" models tried in order. Different providers per entry
+    # so rate limits are uncorrelated -- if one is saturated, the next usually
+    # isn't. Override via [openrouter_models] in the TOML file, or the
+    # LLM_CASCADE_OPENROUTER_MODEL(S) environment variables (env wins).
+    "openrouter_models": [
+        "meta-llama/llama-3.3-70b-instruct:free",
+        "qwen/qwen3-next-80b-a3b-instruct:free",
+        "google/gemma-4-31b-it:free",
+        "nvidia/nemotron-3-super-120b-a12b:free",
+        "nousresearch/hermes-3-llama-3.1-405b:free",
+    ],
+}
+
+
+def _config_file_path() -> Path:
+    """Where to look for the optional TOML config file. Doesn't need to exist."""
+    env_path = os.environ.get("LLM_CASCADE_CONFIG")
+    return Path(env_path).expanduser() if env_path else DEFAULT_CONFIG_PATH
+
+
+def _load_toml_file(path: Path) -> dict:
+    if not path.exists():
+        return {}
+    try:
+        with open(path, "rb") as f:
+            return tomllib.load(f)
+    except (OSError, tomllib.TOMLDecodeError):
+        return {}
+
+
+def _deep_merge(base: dict, override: dict) -> dict:
+    out = dict(base)
+    for key, value in override.items():
+        if isinstance(value, dict) and isinstance(out.get(key), dict):
+            out[key] = _deep_merge(out[key], value)
+        else:
+            out[key] = value
+    return out
+
+
+def log_path() -> Path:
+    """Where the savings ledger (JSONL) is written and read from.
+
+    Override with the ``LLM_CASCADE_LOG_PATH`` environment variable; defaults
+    to ``~/.llm_cascade/log.jsonl``.
+    """
+    override = os.environ.get("LLM_CASCADE_LOG_PATH")
+    return Path(override).expanduser() if override else DEFAULT_LOG_PATH
+
+
+def load_config() -> dict:
+    """Merge built-in defaults with an optional TOML file and return the result.
+
+    Never raises: a missing or unparsable config file is silently treated as
+    "no overrides", since the built-in defaults are a complete, working
+    configuration on their own.
+    """
+    cfg = _deep_merge({}, DEFAULTS)
+    file_cfg = _load_toml_file(_config_file_path())
+    return _deep_merge(cfg, file_cfg)

llm_cascade/pricing.py

@@ -0,0 +1,27 @@
+"""Per-model USD pricing, overridable via the config file (see `config.py`).
+
+This module only affects the numbers reported by `stats()` / `llm-cascade
+--stats` -- it has no bearing on what a provider actually bills you.
+"""
+from __future__ import annotations
+
+from . import config as _config
+
+
+def price_table(cfg: dict | None = None) -> dict[str, tuple[float, float]]:
+    """Model id -> (usd_per_1m_input_tokens, usd_per_1m_output_tokens)."""
+    cfg = cfg if cfg is not None else _config.load_config()
+    return {model: tuple(prices) for model, prices in cfg["pricing"].items()}
+
+
+def model_ids(cfg: dict | None = None) -> dict[str, str]:
+    """Tier name ('opus'/'sonnet'/'haiku') -> concrete Anthropic model id."""
+    cfg = cfg if cfg is not None else _config.load_config()
+    return dict(cfg["models"])
+
+
+def cost_usd(model_id: str, input_tokens: int, output_tokens: int,
+             cfg: dict | None = None) -> float:
+    """Estimated USD cost for one call. Unknown model ids price at $0."""
+    price_in, price_out = price_table(cfg).get(model_id, (0.0, 0.0))
+    return (input_tokens / 1e6) * price_in + (output_tokens / 1e6) * price_out

llm_cascade/providers/__init__.py

@@ -0,0 +1,6 @@
+"""Provider backends for the llm-cascade routing engine.
+
+Each provider module exposes a small, mockable surface (no provider-specific
+types leak out): plain strings and dicts in, plain dicts/strings out. See
+`local_provider.py`, `openrouter_provider.py`, and `anthropic_provider.py`.
+"""

llm_cascade/providers/anthropic_provider.py

@@ -0,0 +1,123 @@
+"""Anthropic provider: the paid, top-of-cascade tier.
+
+The ``anthropic`` SDK is imported lazily (only when this tier is actually
+used) so the library has zero required dependencies. Install it with
+``pip install llm-cascade[anthropic]``.
+"""
+from __future__ import annotations
+
+import os
+
+# Model families that accept adaptive-thinking + effort params. Anything else
+# (e.g. Haiku) gets a plain call with no extra body.
+_ADAPTIVE_MODELS = {"claude-opus-4-8", "claude-sonnet-5"}
+
+_OVERLOAD_MARKERS = (
+    "rate_limit", "overloaded", "quota", "529", "503", "capacity",
+    "too many requests", "model_not_available",
+)
+
+_client = None
+
+
+def _api_key() -> str | None:
+    key = os.environ.get("ANTHROPIC_API_KEY") or os.environ.get("CLAUDE_API_KEY")
+    return key.strip() if key else None
+
+
+def get_client():
+    """Lazily construct (and cache) the Anthropic client.
+
+    Raises a clear, actionable `RuntimeError` if the SDK isn't installed or no
+    API key is configured, instead of letting an SDK-internal error surface.
+    """
+    global _client
+    if _client is None:
+        try:
+            from anthropic import Anthropic
+        except ImportError as e:
+            raise RuntimeError(
+                "The 'anthropic' package is required for paid completions. "
+                "Install it with: pip install llm-cascade[anthropic]"
+            ) from e
+        key = _api_key()
+        if not key:
+            raise RuntimeError(
+                "Missing ANTHROPIC_API_KEY (set the environment variable, or "
+                "CLAUDE_API_KEY as an alias)."
+            )
+        _client = Anthropic(api_key=key)
+    return _client
+
+
+def is_overload(error: Exception) -> bool:
+    """Heuristic: does this exception look like a transient overload (as
+    opposed to a real bug), i.e. worth retrying against the fallback model?"""
+    message = str(error).lower()
+    return any(marker in message for marker in _OVERLOAD_MARKERS)
+
+
+def _extra_body(model_id: str, thinking: str, effort: str | None) -> dict:
+    """Adaptive-thinking / effort params, sent via `extra_body` so this works
+    across SDK versions. Only the adaptive-thinking model family accepts them."""
+    if model_id not in _ADAPTIVE_MODELS:
+        return {}
+    body: dict = {}
+    if thinking == "adaptive":
+        body["thinking"] = {"type": "adaptive"}
+    elif thinking == "off":
+        body["thinking"] = {"type": "disabled"}
+    if effort:
+        body["output_config"] = {"effort": effort}
+    return body
+
+
+def _text(message) -> str:
+    """Concatenate only the text content blocks (skip thinking/tool blocks)."""
+    return "".join(
+        getattr(block, "text", "") for block in message.content
+        if getattr(block, "type", None) == "text"
+    ).strip()
+
+
+def complete(prompt: str, system: str | None, model_id: str, thinking: str,
+             effort: str | None, max_tokens: int,
+             fallback_model_id: str | None = None) -> dict:
+    """Call the given model; on overload, retry once against `fallback_model_id`.
+
+    Returns ``{"text", "model", "input_tokens", "output_tokens"}``. Non-overload
+    errors, and overload errors with no fallback configured, propagate to the
+    caller (the router decides whether to degrade to a free tier).
+    """
+    client = get_client()
+    base = {"model": model_id, "max_tokens": max_tokens,
+            "messages": [{"role": "user", "content": prompt}]}
+    if system:
+        base["system"] = system
+
+    def _call(mid: str):
+        kwargs = {**base, "model": mid}
+        extra_body = _extra_body(mid, thinking, effort)
+        # Streaming for large outputs avoids client-side HTTP timeouts.
+        if max_tokens >= 8000:
+            with client.messages.stream(**kwargs, extra_body=extra_body) as stream:
+                message = stream.get_final_message()
+        else:
+            message = client.messages.create(**kwargs, extra_body=extra_body)
+        return mid, message
+
+    try:
+        used_model, message = _call(model_id)
+    except Exception as e:
+        if fallback_model_id and is_overload(e):
+            used_model, message = _call(fallback_model_id)
+        else:
+            raise
+
+    usage = getattr(message, "usage", None)
+    return {
+        "text": _text(message),
+        "model": used_model,
+        "input_tokens": getattr(usage, "input_tokens", 0) if usage else 0,
+        "output_tokens": getattr(usage, "output_tokens", 0) if usage else 0,
+    }

llm_cascade/providers/local_provider.py

@@ -0,0 +1,104 @@
+"""Local provider: cost-free text generation via a local Ollama server.
+
+This is the bottom of the cascade -- $0, no network egress beyond localhost
+(or wherever ``LLM_CASCADE_OLLAMA_BASE_URL`` points). If Ollama isn't running,
+`is_up()` returns False and the router moves on to the next tier.
+"""
+from __future__ import annotations
+
+import json
+import os
+import urllib.request
+
+DEFAULT_BASE_URL = "http://localhost:11434"
+
+# Lane name -> Ollama model tag. A lane is a "quality knob" independent of the
+# exact model pulled; pass an exact Ollama tag directly as `model=` to bypass
+# lanes entirely.
+DEFAULT_LANES: dict[str, str] = {
+    "fast": "qwen2.5:3b",
+    "quality": "qwen2.5:7b",
+    "creative": "qwen2.5:14b",
+}
+DEFAULT_LANE = "fast"
+
+DEFAULT_OPTIONS: dict[str, float | int] = {"temperature": 0.7, "num_predict": 512}
+
+
+def _base_url() -> str:
+    url = os.environ.get("LLM_CASCADE_OLLAMA_BASE_URL", DEFAULT_BASE_URL)
+    return url.rstrip("/").removesuffix("/v1")
+
+
+def _post(path: str, payload: dict, timeout: int = 240) -> dict:
+    req = urllib.request.Request(
+        f"{_base_url()}{path}",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json"},
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return json.loads(r.read().decode("utf-8"))
+
+
+def is_up() -> bool:
+    """Quick reachability check -- call this before `generate()`/`chat()` so a
+    down Ollama fails fast instead of waiting out a long timeout."""
+    try:
+        urllib.request.urlopen(f"{_base_url()}/api/tags", timeout=4)
+        return True
+    except Exception:
+        return False
+
+
+def installed_models() -> list[str]:
+    """Tags currently pulled in the local Ollama instance (empty list if down)."""
+    try:
+        with urllib.request.urlopen(f"{_base_url()}/api/tags", timeout=5) as r:
+            return [m["name"] for m in json.loads(r.read())["models"]]
+    except Exception:
+        return []
+
+
+def _resolve(model: str | None, lanes: dict[str, str] | None = None) -> str:
+    """Map a lane name ('fast'/'quality'/...) or exact tag to an installed model.
+    Falls back to any installed qwen model, then to whatever is installed, if the
+    requested tag isn't pulled yet."""
+    lane_map = lanes or DEFAULT_LANES
+    want = lane_map.get(model or DEFAULT_LANE, model or lane_map[DEFAULT_LANE])
+    have = installed_models()
+    if want in have:
+        return want
+    for m in have:
+        if "qwen" in m:
+            return m
+    return have[0] if have else want
+
+
+def generate(prompt: str, model: str | None = None, system: str | None = None,
+             options: dict | None = None, timeout: int = 180,
+             lanes: dict[str, str] | None = None) -> str:
+    """Single-shot completion. `model` is a lane name or an exact Ollama tag."""
+    payload = {
+        "model": _resolve(model, lanes),
+        "prompt": prompt,
+        "stream": False,
+        "options": {**DEFAULT_OPTIONS, **(options or {})},
+    }
+    if system:
+        payload["system"] = system
+    return _post("/api/generate", payload, timeout).get("response", "").strip()
+
+
+def chat(prompt: str, model: str | None = None, system: str | None = None,
+         options: dict | None = None, timeout: int = 180,
+         lanes: dict[str, str] | None = None) -> str:
+    """Chat-style completion (same return shape as `generate()`)."""
+    msgs = ([{"role": "system", "content": system}] if system else []) + \
+           [{"role": "user", "content": prompt}]
+    payload = {
+        "model": _resolve(model, lanes),
+        "messages": msgs,
+        "stream": False,
+        "options": {**DEFAULT_OPTIONS, **(options or {})},
+    }
+    return _post("/api/chat", payload, timeout).get("message", {}).get("content", "").strip()

llm_cascade/providers/openrouter_provider.py

@@ -0,0 +1,93 @@
+"""OpenRouter provider: free-tier (``:free`` suffix) models.
+
+Free models are rate-limited upstream (expect occasional 429s), so this tries
+each configured model in order and moves to the next on any error. Returns
+``None`` only if every model fails or no API key is configured -- callers
+treat that as "this tier had nothing, try the next one."
+"""
+from __future__ import annotations
+
+import json
+import os
+import urllib.request
+
+OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
+
+# Tried in order. Different providers per entry so rate limits are
+# uncorrelated -- if one is saturated, the next usually isn't.
+DEFAULT_FREE_MODELS: tuple[str, ...] = (
+    "meta-llama/llama-3.3-70b-instruct:free",
+    "qwen/qwen3-next-80b-a3b-instruct:free",
+    "google/gemma-4-31b-it:free",
+    "nvidia/nemotron-3-super-120b-a12b:free",
+    "nousresearch/hermes-3-llama-3.1-405b:free",
+)
+
+
+def free_models() -> list[str]:
+    """The list of ``:free`` models to try, in order.
+
+    ``LLM_CASCADE_OPENROUTER_MODEL`` pins a single model; the plural
+    ``LLM_CASCADE_OPENROUTER_MODELS`` sets a comma-separated list. Either env
+    var overrides the built-in default.
+    """
+    single = os.environ.get("LLM_CASCADE_OPENROUTER_MODEL")
+    if single:
+        return [single.strip()]
+    raw = os.environ.get("LLM_CASCADE_OPENROUTER_MODELS")
+    if raw:
+        return [m.strip() for m in raw.split(",") if m.strip()]
+    return list(DEFAULT_FREE_MODELS)
+
+
+def _api_key() -> str | None:
+    key = os.environ.get("OPENROUTER_API_KEY")
+    return key.strip() if key else None
+
+
+def _headers(key: str) -> dict:
+    headers = {"Content-Type": "application/json", "Authorization": f"Bearer {key}"}
+    referer = os.environ.get("LLM_CASCADE_HTTP_REFERER")
+    title = os.environ.get("LLM_CASCADE_APP_TITLE")
+    if referer:
+        headers["HTTP-Referer"] = referer
+    if title:
+        headers["X-Title"] = title
+    return headers
+
+
+def complete(prompt: str, system: str | None = None, max_tokens: int | None = None,
+             models: list[str] | None = None) -> dict | None:
+    """Try each free model in order; return the first success.
+
+    Returns ``{"text", "model", "input_tokens", "output_tokens"}``, or ``None``
+    if there's no API key or every model in the list failed.
+    """
+    key = _api_key()
+    if not key:
+        return None
+    msgs = ([{"role": "system", "content": system}] if system else []) + \
+           [{"role": "user", "content": prompt}]
+    headers = _headers(key)
+    for model in (models if models is not None else free_models()):
+        body = {"model": model, "messages": msgs, "max_tokens": max_tokens or 1500}
+        req = urllib.request.Request(
+            OPENROUTER_URL, data=json.dumps(body).encode("utf-8"), headers=headers
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=90) as r:
+                data = json.loads(r.read().decode("utf-8"))
+            choice = (data.get("choices") or [{}])[0]
+            text = (choice.get("message", {}).get("content") or "").strip()
+            if not text:
+                continue
+            usage = data.get("usage") or {}
+            return {
+                "text": text,
+                "model": model,
+                "input_tokens": int(usage.get("prompt_tokens", 0) or 0),
+                "output_tokens": int(usage.get("completion_tokens", 0) or 0),
+            }
+        except Exception:
+            continue  # 429 / invalid slug / timeout -> try the next free model
+    return None

llm_cascade/router.py

@@ -0,0 +1,277 @@
+"""Core routing engine: decides which tier handles a task, then runs the
+free -> paid cascade with automatic fallback.
+
+Tier ladder (cheapest to strongest):
+
+    trivial  -> local Ollama only ($0), no cloud fallback at all
+    local    -> local Ollama ($0) -> OpenRouter free model ($0) -> paid Anthropic
+                (safety net) -- mechanical NLP that a small local model handles fine
+    cheap    -> OpenRouter free model ($0) -> paid Anthropic (manual-only tier;
+                no task routes here by default, invoke with task="cheap")
+    light    -> paid Anthropic (Sonnet); OpenRouter free is tried first only if
+                LLM_CASCADE_LIGHT_FREE=1 (opt-in, since this tier is usually
+                customer-facing copy where quality matters more than $0)
+    heavy    -> paid Anthropic (Opus), adaptive thinking, high effort -- real
+                reasoning, long-form, code, review
+
+Edit `task_routes` in a TOML config file (see `config.py`) to calibrate which
+task maps to which tier -- that's the one knob most users need to touch.
+"""
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+from . import config as _config
+from . import pricing as _pricing
+from .providers import anthropic_provider, local_provider, openrouter_provider
+
+# Tier mechanics:
+#   local=True         -> pure local_provider ($0), no cloud at all.
+#   prefer_local=True  -> try local_provider ($0) first; cloud only if it fails.
+#   cheap_cloud=True   -> try OpenRouter free ($0) before paid cloud.
+#   cheap_cloud="env"  -> same, but opt-in via LLM_CASCADE_LIGHT_FREE=1.
+#   lane               -> which local_provider lane to use (fast/quality/creative).
+#   thinking           -> "adaptive" | "off".
+#   effort             -> low|medium|high|xhigh|max.
+#   fallback           -> alternate Anthropic model tier name if the primary overloads.
+TIERS: dict[str, dict] = {
+    "trivial": {"local": True, "lane": "quality"},
+    "local": {"prefer_local": True, "lane": "quality", "cheap_cloud": True,
+              "model": "sonnet", "thinking": "off", "effort": "low",
+              "max_tokens": 2000, "fallback": "haiku"},
+    "cheap": {"cheap_cloud": True,
+              "model": "sonnet", "thinking": "off", "effort": "low",
+              "max_tokens": 3000, "fallback": "haiku"},
+    "light": {"cheap_cloud": "env",
+              "model": "sonnet", "thinking": "off", "effort": "low",
+              "max_tokens": 3000, "fallback": "haiku"},
+    "heavy": {"model": "opus", "thinking": "adaptive", "effort": "high",
+              "max_tokens": 16000, "fallback": "sonnet"},
+}
+TIER_ORDER = ("trivial", "local", "cheap", "light", "heavy")
+
+
+@dataclass
+class Completion:
+    """Result of a routed completion."""
+
+    text: str
+    tier: str
+    model: str
+    cost_usd: float
+    cached: bool = False
+
+
+def pick(task_or_tier: str, cfg: dict | None = None) -> dict:
+    """Resolve a TASK name or TIER name to its tier config (+ `model_id`).
+
+    `task_or_tier` may be an exact tier name ('trivial'/'local'/'cheap'/
+    'light'/'heavy') or any task name from the `task_routes` config; unknown
+    task names fall back to `default_task_tier`.
+    """
+    cfg = cfg if cfg is not None else _config.load_config()
+    default_tier = cfg["default_task_tier"]
+    key = (task_or_tier or default_tier).strip().lower()
+    tier = key if key in TIERS else cfg["task_routes"].get(key, default_tier)
+    tier_cfg = dict(TIERS[tier])
+    tier_cfg["tier"] = tier
+    if not tier_cfg.get("local"):
+        tier_cfg["model_id"] = _pricing.model_ids(cfg)[tier_cfg["model"]]
+    return tier_cfg
+
+
+def _cheap_cloud_on(tier_cfg: dict) -> bool:
+    if os.environ.get("LLM_CASCADE_NO_FREE_CLOUD"):  # global kill-switch
+        return False
+    cheap_cloud = tier_cfg.get("cheap_cloud")
+    if cheap_cloud is True:
+        return True
+    if cheap_cloud == "env":
+        return bool(os.environ.get("LLM_CASCADE_LIGHT_FREE"))
+    return False
+
+
+def _log_call(model: str, input_tokens: int, output_tokens: int, cost: float) -> None:
+    """One line per call, paid or free (cost=0.0). Feeds `stats()`."""
+    path = _config.log_path()
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(json.dumps({
+                "ts": datetime.now(timezone.utc).isoformat(),
+                "model": model,
+                "in": input_tokens,
+                "out": output_tokens,
+                "cost_usd": round(cost, 6),
+            }) + "\n")
+    except OSError:
+        pass
+
+
+def _try_local(prompt: str, lane: str | None, system: str | None) -> str | None:
+    """Try the local tier; None if Ollama is down or returns empty output."""
+    if not local_provider.is_up():
+        return None
+    try:
+        out = local_provider.generate(prompt, model=lane, system=system)
+        return out if (out and out.strip()) else None
+    except Exception:
+        return None
+
+
+def _try_openrouter(prompt: str, system: str | None, max_tokens: int | None) -> dict | None:
+    result = openrouter_provider.complete(prompt, system=system, max_tokens=max_tokens)
+    if result:
+        _log_call("openrouter:" + result["model"],
+                   result["input_tokens"], result["output_tokens"], 0.0)
+    return result
+
+
+def _call_claude(prompt: str, system: str | None, model_id: str, thinking: str,
+                  effort: str | None, max_tokens: int, fallback_tier_model: str | None,
+                  cfg: dict) -> dict:
+    fallback_id = _pricing.model_ids(cfg).get(fallback_tier_model) if fallback_tier_model else None
+    result = anthropic_provider.complete(prompt, system, model_id, thinking, effort,
+                                          max_tokens, fallback_id)
+    cost = _pricing.cost_usd(result["model"], result["input_tokens"], result["output_tokens"], cfg)
+    _log_call(result["model"], result["input_tokens"], result["output_tokens"], cost)
+    result["cost_usd"] = cost
+    return result
+
+
+def route(task: str, prompt: str, system: str | None = None,
+          max_tokens: int | None = None, effort: str | None = None) -> Completion:
+    """Route `task` to the right tier and run the free -> paid cascade.
+
+    `task` may be a task name (see the `task_routes` config) or a tier name
+    directly ('trivial'/'local'/'cheap'/'light'/'heavy'). `max_tokens`/`effort`
+    override the tier's defaults. Returns a `Completion` with the text, the
+    tier and model that actually served it, and the USD cost (0.0 for local
+    or free-cloud tiers).
+    """
+    cfg = _config.load_config()
+    tier_cfg = pick(task, cfg)
+    tier = tier_cfg["tier"]
+
+    if tier_cfg.get("local"):  # pure local ($0), no cloud at all
+        text = local_provider.generate(prompt, model=tier_cfg.get("lane"), system=system)
+        return Completion(text=text, tier=tier, model="local:ollama", cost_usd=0.0)
+
+    if tier_cfg.get("prefer_local"):  # 1) local $0
+        text = _try_local(prompt, tier_cfg.get("lane"), system)
+        if text:
+            return Completion(text=text, tier=tier, model="local:ollama", cost_usd=0.0)
+
+    if _cheap_cloud_on(tier_cfg):  # 2) free cloud $0 (OpenRouter :free)
+        result = _try_openrouter(prompt, system, max_tokens or tier_cfg.get("max_tokens"))
+        if result:
+            return Completion(text=result["text"], tier=tier,
+                               model="openrouter:" + result["model"], cost_usd=0.0)
+
+    try:
+        result = _call_claude(  # 3) paid cloud (quality / safety net)
+            prompt, system, tier_cfg["model_id"], tier_cfg["thinking"],
+            effort or tier_cfg["effort"], max_tokens or tier_cfg["max_tokens"],
+            tier_cfg.get("fallback"), cfg,
+        )
+        return Completion(text=result["text"], tier=tier, model=result["model"],
+                           cost_usd=result["cost_usd"])
+    except Exception as e:
+        # Final safety net: the paid API is unreachable for ANY reason (bad or
+        # revoked key, no credits, network down, provider outage -- not just
+        # overload) -- degrade to the free tiers instead of raising, so
+        # automated/scheduled callers never break outright.
+        _log_call("degraded:" + type(e).__name__, 0, 0, 0.0)
+        if not _cheap_cloud_on(tier_cfg):  # didn't try free cloud yet this call
+            result = _try_openrouter(prompt, system, max_tokens or tier_cfg.get("max_tokens"))
+            if result:
+                return Completion(text=result["text"], tier=tier,
+                                   model="openrouter:" + result["model"], cost_usd=0.0)
+        text = _try_local(prompt, tier_cfg.get("lane"), system)
+        if text:
+            return Completion(text=text, tier=tier, model="local:ollama", cost_usd=0.0)
+        raise
+
+
+def complete(prompt: str, task: str = "light", system: str | None = None,
+             max_tokens: int | None = None, effort: str | None = None) -> str:
+    """Same cascade as `route()`, returning just the completion text."""
+    return route(task, prompt, system=system, max_tokens=max_tokens, effort=effort).text
+
+
+def complete_json(prompt: str, task: str = "extract", system: str | None = None, **kw) -> dict:
+    """Like `complete()`, but instructs the model to reply with one JSON object
+    and parses it."""
+    sysj = (system or "") + ("\n\nReply with ONE valid JSON object only. "
+                              "No markdown, no ```json fences, no extra text.")
+    raw = complete(prompt, task=task, system=sysj, **kw).strip()
+    if raw.startswith("```"):
+        raw = raw.strip("`")
+        raw = raw.split("\n", 1)[1] if "\n" in raw else raw
+    start, end = raw.find("{"), raw.rfind("}")
+    if start >= 0 and end > start:
+        raw = raw[start:end + 1]
+    return json.loads(raw)
+
+
+def stats() -> dict:
+    """Cumulative cost + free (local/OpenRouter) vs paid (Anthropic) call counts,
+    read from the savings ledger (see `config.log_path()`)."""
+    path = _config.log_path()
+    paid = free = 0
+    cost = 0.0
+    by_model: dict[str, int] = {}
+    if path.exists():
+        for line in path.read_text(encoding="utf-8").splitlines():
+            try:
+                entry = json.loads(line)
+            except Exception:
+                continue
+            call_cost = float(entry.get("cost_usd", 0) or 0)
+            cost += call_cost
+            if call_cost > 0:
+                paid += 1
+            else:
+                free += 1
+            model = entry.get("model", "unknown")
+            by_model[model] = by_model.get(model, 0) + 1
+    return {
+        "paid_calls": paid,
+        "free_calls": free,
+        "total_cost_usd": round(cost, 4),
+        "by_model": by_model,
+    }
+
+
+def _tier_destination(tier: str, cfg: dict) -> str:
+    """Human-readable description of a tier's provider chain, for `--routes`."""
+    tier_cfg = pick(tier, cfg)
+    if tier_cfg.get("local"):
+        return "local (Ollama, $0)"
+    chain = []
+    if tier_cfg.get("prefer_local"):
+        chain.append("local (Ollama, $0)")
+    cheap_cloud = tier_cfg.get("cheap_cloud")
+    if cheap_cloud is True:
+        chain.append("OpenRouter free ($0)")
+    elif cheap_cloud == "env":
+        chain.append("OpenRouter free ($0, if LLM_CASCADE_LIGHT_FREE=1)")
+    chain.append(tier_cfg["model_id"] + (" (safety net)" if chain else ""))
+    return " -> ".join(chain)
+
+
+def describe_routes(cfg: dict | None = None) -> str:
+    """Render the tier ladder + task -> tier table (used by `llm-cascade --routes`)."""
+    cfg = cfg if cfg is not None else _config.load_config()
+    lines = []
+    for tier in TIER_ORDER:
+        tasks = sorted(t for t, v in cfg["task_routes"].items() if v == tier)
+        label = tier + (" (manual)" if tier == "cheap" else "")
+        line = f"[{label:14}] -> {_tier_destination(tier, cfg)}"
+        if tasks:
+            line += f"\n                 {', '.join(tasks)}"
+        lines.append(line)
+    return "\n".join(lines)

llm_cascade-0.1.0.dist-info/METADATA

@@ -0,0 +1,196 @@
+Metadata-Version: 2.4
+Name: llm-cascade
+Version: 0.1.0
+Summary: Free-first LLM routing for Python: local Ollama -> OpenRouter free models -> paid Anthropic API, with automatic fallback and a savings ledger.
+Project-URL: Homepage, https://github.com/luandv92/llm-cascade
+Project-URL: Repository, https://github.com/luandv92/llm-cascade
+Project-URL: Issues, https://github.com/luandv92/llm-cascade/issues
+Project-URL: Changelog, https://github.com/luandv92/llm-cascade/blob/main/CHANGELOG.md
+Author: luandv92
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,anthropic,claude,cost-optimization,llm,ollama,openrouter,router
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: tomli>=2.0.1; python_version < '3.11'
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.40.0; extra == 'anthropic'
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# llm-cascade
+
+[![CI](https://github.com/luandv92/llm-cascade/actions/workflows/ci.yml/badge.svg)](https://github.com/luandv92/llm-cascade/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/llm-cascade.svg)](https://pypi.org/project/llm-cascade/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+[![Python versions](https://img.shields.io/pypi/pyversions/llm-cascade.svg)](https://pypi.org/project/llm-cascade/)
+
+Free-first LLM routing for Python: local Ollama ($0) -> OpenRouter free models
+($0) -> paid Anthropic API, with automatic fallback and a savings ledger.
+Zero required dependencies.
+
+## Why
+
+Most teams route every prompt to the same frontier model regardless of how
+hard the task actually is, and the API bill grows with usage instead of with
+value delivered. llm-cascade flips that: it tries the cheapest tier that can
+plausibly do the job first, and only escalates when that tier is unreachable
+or genuinely too weak for the task. In production, that means classification,
+extraction, and summarization run for $0 on a local model or a free cloud
+model, while code review, planning, and long-form writing still get a
+frontier model -- and if the paid API is ever down or out of credits, your
+scheduled jobs degrade to a free tier instead of crashing.
+
+## Install
+
+```bash
+pip install llm-cascade
+```
+
+The base package has **zero required third-party dependencies**. To use the
+paid Anthropic tier, install the extra:
+
+```bash
+pip install "llm-cascade[anthropic]"
+```
+
+Without it, the local (Ollama) and free-cloud (OpenRouter) tiers still work
+end to end -- you'll only see an error if a task actually routes to the paid
+tier and the SDK isn't installed.
+
+## Quickstart
+
+```python
+from llm_cascade import route
+
+result = route("summarize", "Paste a long document here...")
+print(result.text)        # the completion
+print(result.tier)        # which tier served it: trivial/local/cheap/light/heavy
+print(result.model)       # e.g. "local:ollama", "openrouter:qwen/...", "claude-sonnet-5"
+print(result.cost_usd)    # 0.0 for local/free-cloud tiers
+```
+
+Or from the command line:
+
+```bash
+llm-cascade "Summarize this text: ..." --task summarize
+llm-cascade --routes   # show the tier ladder and task -> tier table
+llm-cascade --stats    # show cumulative cost and free-vs-paid call counts
+```
+
+## How routing works
+
+Each call resolves a **task name** (or a tier name directly) to a **tier**,
+then runs that tier's provider chain in order until one succeeds:
+
+| Tier      | Chain                                            | Use for                                  |
+|-----------|---------------------------------------------------|-------------------------------------------|
+| `trivial` | local Ollama only ($0)                            | high-volume micro-tasks, never worth the network |
+| `local`   | local Ollama ($0) -> OpenRouter free ($0) -> paid  | mechanical NLP: classify, extract, summarize, translate |
+| `cheap`   | OpenRouter free ($0) -> paid                       | manual opt-in tier, invoke directly with `task="cheap"` |
+| `light`   | paid (Sonnet); free-cloud only if `LLM_CASCADE_LIGHT_FREE=1` | customer-facing copy that needs reliable polish |
+| `heavy`   | paid (Opus), adaptive thinking, high effort        | real reasoning: code, review, planning, long-form |
+
+If the paid API call fails for *any* reason -- bad or revoked key, no
+credits, a provider outage, a network blip -- llm-cascade degrades to the
+free tiers instead of raising, so scheduled or automated callers don't break
+outright. It only raises once every tier has been tried and failed.
+
+The default task -> tier table:
+
+```
+classify, extract, summarize, translate, chat  -> local
+rewrite                                        -> light
+code, review, plan, creative                   -> heavy
+```
+
+Anything not in that table falls back to `default_task_tier` (`light` by
+default). See `config/routes.example.toml` for the full, commented schema.
+
+## Configuration
+
+Configuration is resolved with this precedence (lowest to highest):
+
+1. Built-in defaults.
+2. An optional TOML file: `~/.llm_cascade/routes.toml`, or the path given in
+   `LLM_CASCADE_CONFIG`.
+3. Environment variables, for the handful of settings that make sense as env
+   vars.
+
+Copy [`config/routes.example.toml`](config/routes.example.toml) to
+`~/.llm_cascade/routes.toml` to add your own task names, retune which tier
+handles a task, or override model ids and pricing.
+
+Environment variables:
+
+| Variable | Purpose |
+|---|---|
+| `ANTHROPIC_API_KEY` | Enables the paid tier (`light`/`heavy`, and the final safety net). |
+| `OPENROUTER_API_KEY` | Enables the free-cloud tier (OpenRouter `:free` models). |
+| `LLM_CASCADE_CONFIG` | Path to a TOML config file, instead of `~/.llm_cascade/routes.toml`. |
+| `LLM_CASCADE_LOG_PATH` | Path to the savings ledger, instead of `~/.llm_cascade/log.jsonl`. |
+| `LLM_CASCADE_LIGHT_FREE` | Set to `1` to try free-cloud first on the `light` tier too. |
+| `LLM_CASCADE_NO_FREE_CLOUD` | Set to any value to disable the free-cloud tier globally. |
+| `LLM_CASCADE_HTTP_REFERER` / `LLM_CASCADE_APP_TITLE` | Sent as `HTTP-Referer` / `X-Title` on OpenRouter requests (optional; omitted if unset). |
+| `LLM_CASCADE_OPENROUTER_MODEL` / `LLM_CASCADE_OPENROUTER_MODELS` | Override the free model (singular) or ordered list (comma-separated) tried on OpenRouter. |
+| `LLM_CASCADE_OLLAMA_BASE_URL` | Ollama server URL (default `http://localhost:11434`). |
+
+See [`.env.example`](.env.example) for a copy-pasteable starting point.
+
+## Savings ledger
+
+Every call -- free or paid -- appends one line to a JSONL ledger
+(`~/.llm_cascade/log.jsonl` by default). `stats()` / `llm-cascade --stats`
+summarize it:
+
+```bash
+$ llm-cascade --stats
+{
+  "paid_calls": 3,
+  "free_calls": 41,
+  "total_cost_usd": 0.0842,
+  "by_model": {
+    "local:ollama": 30,
+    "openrouter:qwen/qwen3-next-80b-a3b-instruct:free": 11,
+    "claude-sonnet-5": 3
+  }
+}
+```
+
+## Comparison
+
+Unlike heavier multi-provider routers, llm-cascade ships with zero required
+dependencies and a single opinionated cascade -- it's a routing policy you can
+read in one file, not a platform.
+
+## Contributing
+
+Issues and pull requests are welcome at
+[github.com/luandv92/llm-cascade](https://github.com/luandv92/llm-cascade).
+Before opening a PR:
+
+```bash
+pip install -e ".[dev,anthropic]"
+ruff check .
+pytest tests/ -v
+```
+
+Tests never touch real networks -- Ollama, OpenRouter, and the Anthropic SDK
+are all mocked (see `tests/conftest.py`).
+
+## License
+
+MIT -- see [LICENSE](LICENSE).

llm_cascade-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+llm_cascade/__init__.py,sha256=I2h8gApJoagXoGf2ma8rksIGPXu1hdecWgEcAcZb2A4,719
+llm_cascade/__main__.py,sha256=t8oO-Xt48El7YXnfdGBKc4uhW9CAHoizIJir09kT8XQ,203
+llm_cascade/cli.py,sha256=sWFRRsn9N2CJtY5NDSY1FfHq8wrZ_cYfpl2AyJsHQGo,2424
+llm_cascade/config.py,sha256=Z1OLgxW7YHC1D9YP3ooDpKebYHfeLLvfcPZGSArT82Q,4399
+llm_cascade/pricing.py,sha256=oVZ5vP2FBWLiUrr7AEXHt5Tg0kcoAGSDAdKwLSxxWh8,1152
+llm_cascade/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+llm_cascade/router.py,sha256=dbojoCmn1FnIL57s4Bw36L-b-deKselfWXfKmdP1qN4,11966
+llm_cascade/providers/__init__.py,sha256=4RYZkRdu8Kj67TlLjRrgDMWj6mLpywWYmfMVaEDIPJQ,289
+llm_cascade/providers/anthropic_provider.py,sha256=frvij5gtFC45oS22KxQVPauG-_nlU3nkEvhOaQALu4k,4427
+llm_cascade/providers/local_provider.py,sha256=XnuQETyZoXZ4DSeO5yyT0xtM68_t6aAHqjuAY2RAwAQ,3769
+llm_cascade/providers/openrouter_provider.py,sha256=pTafYSvS-qpsnfF_vaCUn82SQ47cCIt2OFLL0YwF67Y,3508
+llm_cascade-0.1.0.dist-info/METADATA,sha256=8S3Bl3sKZWKqWl-WSH4D7cJm2vulp8CdKVL_tIyob4U,8124
+llm_cascade-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+llm_cascade-0.1.0.dist-info/entry_points.txt,sha256=4VrpGvSjuzKOD5Am33zWgYy9sxwUyAKhm2Tzth-BpoY,53
+llm_cascade-0.1.0.dist-info/licenses/LICENSE,sha256=Kh3ppltVP61oAGqAk9UZUXcy8AsIdkfjZ5msTSsNWfI,1065
+llm_cascade-0.1.0.dist-info/RECORD,,

llm_cascade-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

llm_cascade-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+llm-cascade = llm_cascade.cli:main

llm_cascade-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 luandv92
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.