athena-llm 0.2.0__tar.gz

athena_llm-0.2.0/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: athena-llm
+Version: 0.2.0
+Summary: Athena LLM client — one seam across frontier, hosted-open (with failover), and local backends
+Requires-Python: >=3.10
+Requires-Dist: anthropic>=0.40.0
+Requires-Dist: httpx>=0.27

athena_llm-0.2.0/athena_llm.egg-info/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: athena-llm
+Version: 0.2.0
+Summary: Athena LLM client — one seam across frontier, hosted-open (with failover), and local backends
+Requires-Python: >=3.10
+Requires-Dist: anthropic>=0.40.0
+Requires-Dist: httpx>=0.27

athena_llm-0.2.0/athena_llm.egg-info/SOURCES.txt

@@ -0,0 +1,7 @@
+athena_llm.py
+pyproject.toml
+athena_llm.egg-info/PKG-INFO
+athena_llm.egg-info/SOURCES.txt
+athena_llm.egg-info/dependency_links.txt
+athena_llm.egg-info/requires.txt
+athena_llm.egg-info/top_level.txt

athena_llm-0.2.0/athena_llm.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

athena_llm-0.2.0/athena_llm.egg-info/requires.txt

@@ -0,0 +1,2 @@
+anthropic>=0.40.0
+httpx>=0.27

athena_llm-0.2.0/athena_llm.egg-info/top_level.txt

@@ -0,0 +1 @@
+athena_llm

athena_llm-0.2.0/athena_llm.py

@@ -0,0 +1,444 @@
+"""Athena LLM client — unified async completion across a frontier API (Anthropic),
+hosted-open providers (DeepInfra / Together), and the local Ollama service on MS-01.
+
+Backend selection is per call, so "use an open model where it makes sense" is a
+config choice, not a rewrite. Its only dependencies are `anthropic` and `httpx`,
+which the AI projects already have — a framework was rejected on measurement
+(see `docs/provider-redundancy.md`).
+
+**This module is the seam.** Every model call in a project goes through `complete()`,
+so the internals here can change — a second provider, a different library, a gateway —
+without touching a single call site. That is what makes the cheap implementation the
+correct one: it stays reversible.
+
+Local inference keeps data on MS-01 (consistent with the MCP data-boundary rule).
+
+Env:
+  ATHENA_OLLAMA_URL     default http://127.0.0.1:11434   (localhost-only by design)
+  ATHENA_TIMEOUT        default 120 (seconds)
+  ATHENA_HOSTED_CHAIN   default "deepinfra,together" — ordered hosted-open failover
+  DEEPINFRA_API_KEY / TOGETHER_API_KEY   per-provider keys (per-app, never shared)
+
+Usage:
+    from athena_llm import complete
+    text = await complete(system, user, backend="deepinfra",
+                          model="deepseek-ai/DeepSeek-V4-Pro", max_tokens=400)
+    # backend="anthropic", model="claude-sonnet-5"  -> frontier path
+    # backend="ollama",    model="qwen2.5:7b"       -> on-box path
+"""
+
+import asyncio
+import base64
+import json
+import logging
+import mimetypes
+import os
+import re
+import time
+
+import anthropic
+import httpx
+
+log = logging.getLogger(__name__)
+
+OLLAMA_URL = os.environ.get("ATHENA_OLLAMA_URL", "http://127.0.0.1:11434")
+DEFAULT_TIMEOUT = float(os.environ.get("ATHENA_TIMEOUT", "120"))
+
+# Hosted-open providers, all OpenAI-compatible → one code path, no new dep.
+# A second provider is not a nicety: a migration off frontier fixes cap-exhaustion and
+# REBUILDS single-vendor risk at a new address. Same weights, different vendor, so a
+# failover changes the address and not the answers.
+# Kept as module-level names for back-compat: callers that predate the provider chain
+# import DEEPINFRA_URL directly (e.g. a path that POSTs to the vendor itself rather than
+# going through complete()). Such a caller is OUTSIDE the seam and gets no failover.
+DEEPINFRA_URL = os.environ.get("DEEPINFRA_URL", "https://api.deepinfra.com/v1/openai")
+TOGETHER_URL = os.environ.get("TOGETHER_URL", "https://api.together.xyz/v1")
+
+HOSTED_PROVIDERS = {
+    "deepinfra": {
+        "url": DEEPINFRA_URL,
+        "probe_url": "https://api.deepinfra.com/v1/openai",  # canonical — see _probe_provider
+        "key_var": "DEEPINFRA_API_KEY",
+        # fail_fast: when DeepInfra has no capacity, reject INSTANTLY with 429 instead of
+        # holding the connection open. Measured on BidScout 2026-08-04: without it, latencies
+        # were 60s(timeout) / 27s / 60s(timeout) / 18s; with it, 0.1-0.2s every time. That
+        # matters far more here than there, because _hosted_once retries 3x at DEFAULT_TIMEOUT
+        # (120s) — a stalled provider burned ~364s BEFORE the chain ever tried Together, so the
+        # second source could not do its job. A rejected request never reaches the model and is
+        # not billed, and a healthy DeepInfra serves normally, so there is no downside.
+        "quirks": {"fail_fast": True},
+    },
+    "together": {
+        "url": TOGETHER_URL,
+        "probe_url": "https://api.together.xyz/v1",
+        "key_var": "TOGETHER_API_KEY",
+        # Together serves DeepSeek-V4-Pro in REASONING mode by default: the whole max_tokens
+        # budget is spent on reasoning_tokens and `content` comes back EMPTY with
+        # finish_reason=length. DeepInfra returns content for the identical request. Same
+        # weights, different serving config — normalize it, or a failover silently answers
+        # nothing. Both providers accept this param.
+        "quirks": {"reasoning_effort": "none"},
+    },
+}
+# Ordered failover chain. A provider with no key configured is skipped, so a project
+# that hasn't set up a second vendor keeps today's single-provider behaviour exactly.
+HOSTED_CHAIN = [p.strip() for p in
+                os.environ.get("ATHENA_HOSTED_CHAIN", "deepinfra,together").split(",")
+                if p.strip()]
+
+_TRANSIENT_HTTP = (408, 409, 425, 429, 500, 502, 503, 504, 529)
+_PROBE_TTL = 60.0
+_probe_cache: dict[str, tuple[float, str]] = {}
+
+
+def loads_lenient(text: str):
+    """`json.loads` for hosted-open output. Frontier models return clean JSON; hosted-open
+    ones wrap it in ```code fences``` and leak raw control chars even in json mode — which
+    vanilla `json.loads` rejects (fleet gotcha, LESSONS 2026-07-19). Strips fences, tolerates
+    control chars (`strict=False`), falls back to the first {...}/[...] block. Raises on total
+    failure so a real parse problem still surfaces.
+
+    Also strips a leading Qwen3 `<think>…</think>` block, which the model emits (often empty)
+    even with `/no_think`."""
+    s = (text or "").strip()
+    s = re.sub(r"^<think>.*?</think>", "", s, count=1, flags=re.DOTALL).strip()
+    if s.startswith("```"):
+        s = re.sub(r"^```[a-zA-Z0-9]*\s*", "", s)
+        s = re.sub(r"\s*```$", "", s).strip()
+    try:
+        return json.loads(s, strict=False)
+    except json.JSONDecodeError:
+        m = re.search(r"[\{\[].*[\}\]]", s, re.DOTALL)
+        if m:
+            return json.loads(m.group(0), strict=False)
+        raise
+
+
+_anthropic_client: anthropic.AsyncAnthropic | None = None
+
+
+def _anthropic() -> anthropic.AsyncAnthropic:
+    """Client for the frontier (Anthropic) path.
+
+    Every BULK/BATCH caller flows through here, so this is the single place to isolate their
+    spend: set ``ANTHROPIC_SCORING_API_KEY`` to a SEPARATE workspace with its own cap, and the
+    whole batch bills there. That workspace hitting its cap then only stalls batch work — it
+    can never starve on-demand user analysis, which bills to ``ANTHROPIC_API_KEY`` through its
+    own client. Unset → falls back to ``ANTHROPIC_API_KEY`` (unchanged single-key behaviour)."""
+    global _anthropic_client
+    if _anthropic_client is None:
+        key = os.environ.get("ANTHROPIC_SCORING_API_KEY") or os.environ.get("ANTHROPIC_API_KEY")
+        _anthropic_client = anthropic.AsyncAnthropic(api_key=key) if key else anthropic.AsyncAnthropic()
+    return _anthropic_client
+
+
+def eval_anthropic_client() -> anthropic.AsyncAnthropic:
+    """Client for the frontier arm of an A/B or any eval — NEVER the production key.
+
+    A bare ``anthropic.AsyncAnthropic()`` inherits ``ANTHROPIC_API_KEY``, i.e. the live
+    product's workspace and cap; an eval through it can (and once did) trip prod's cap and take
+    the paid product down. Evals bill to a dedicated ``ANTHROPIC_EVAL_API_KEY``. Hard-fails when
+    unset rather than silently falling back — an eval that can't isolate its spend must not run."""
+    key = os.environ.get("ANTHROPIC_EVAL_API_KEY")
+    if not key:
+        raise RuntimeError(
+            "ANTHROPIC_EVAL_API_KEY is not set. Evals must bill to a dedicated eval workspace, "
+            "never ANTHROPIC_API_KEY (prod) — set an eval key or don't run the frontier arm."
+        )
+    return anthropic.AsyncAnthropic(api_key=key)
+
+
+def _encode_image(item) -> tuple[str, str]:
+    """Return (base64_data, media_type) for one image.
+
+    `item` may be a filesystem path (str), raw bytes, or a (bytes|path, media_type) pair.
+    Rasterized PDF pages (`pdftoppm -png`) are PNG, so that's the default when the type
+    can't be inferred."""
+    media_type = None
+    if isinstance(item, tuple):
+        item, media_type = item
+    if isinstance(item, (bytes, bytearray)):
+        data = bytes(item)
+    else:  # filesystem path
+        with open(item, "rb") as fh:
+            data = fh.read()
+        media_type = media_type or mimetypes.guess_type(str(item))[0]
+    return base64.standard_b64encode(data).decode("ascii"), media_type or "image/png"
+
+
+async def _probe_provider(provider: str) -> str:
+    """Is the provider that just failed actually reachable? -> healthy|down|unknown.
+
+    A failover fires on a SYMPTOM — timeout, dropped connection, 5xx — and those are identical
+    whether the vendor is down or the fault is local (a restart, the container network, a bad
+    base URL, a dead key). Unattended, a silent failover hides our own bugs: the product keeps
+    answering and nobody learns the primary is broken. `healthy` is the interesting answer.
+
+    Probes the provider's CANONICAL endpoint, never the configured one: if the failure was a
+    bad base URL, probing that same bad URL would report "down" and blame the vendor for our
+    own misconfiguration — the exact misattribution this is here to prevent.
+
+    GET /models only — no inference, no spend. Cached so a burst can't become a probe storm."""
+    spec = HOSTED_PROVIDERS.get(provider)
+    if not spec:
+        return "unknown"
+    key = os.environ.get(spec["key_var"])
+    if not key:
+        return "unknown"
+
+    now = time.monotonic()
+    cached = _probe_cache.get(provider)
+    if cached and now - cached[0] < _PROBE_TTL:
+        return cached[1]
+
+    try:
+        async with httpx.AsyncClient(timeout=5.0) as client:
+            r = await client.get(f"{spec.get('probe_url') or spec['url']}/models",
+                                 headers={"Authorization": f"Bearer {key}"})
+        # 2xx = up and our key works. 401/403 = up, but OUR key is the problem — still
+        # "healthy" for attribution: the vendor isn't down, we are.
+        verdict = "healthy" if r.status_code < 500 else "down"
+    except Exception:
+        verdict = "down"
+
+    _probe_cache[provider] = (now, verdict)
+    return verdict
+
+
+async def _hosted_once(provider: str, payload: dict, timeout: float | None):
+    """One hosted-open provider, up to 3 attempts. Retries transient failures only — a single
+    load spike must not kill a whole analysis, but a 4xx (bad request, bad model, dead key) is
+    a config problem and is raised immediately rather than retried into a failover that hides it."""
+    spec = HOSTED_PROVIDERS[provider]
+    key = os.environ[spec["key_var"]]
+    payload = {**payload, **(spec.get("quirks") or {})}
+    last_exc: Exception | None = None
+    for attempt in range(3):
+        try:
+            async with httpx.AsyncClient(timeout=timeout or DEFAULT_TIMEOUT) as client:
+                r = await client.post(
+                    f"{spec['url']}/chat/completions",
+                    headers={"Authorization": f"Bearer {key}"},
+                    json=payload,
+                )
+            if r.status_code in _TRANSIENT_HTTP:
+                last_exc = RuntimeError(f"{provider} transient HTTP {r.status_code}")
+                if attempt < 2:
+                    await asyncio.sleep(1.5 * (attempt + 1))
+                    continue
+                raise last_exc
+            r.raise_for_status()
+            data = r.json()
+            # An HTTP 200 with EMPTY content is a failure, not an answer — never hand the caller
+            # "". Seen for real and intermittently: a provider serving the model in reasoning
+            # mode spends the whole budget before writing a word (finish_reason=length). It
+            # recurs sporadically even with reasoning disabled, so retry here first and only
+            # then let the chain fail over — a retry is cheaper than a vendor switch.
+            choice = (data.get("choices") or [{}])[0]
+            if not ((choice.get("message") or {}).get("content") or "").strip():
+                last_exc = RuntimeError(
+                    f"{provider} returned empty content "
+                    f"(finish_reason={choice.get('finish_reason')})")
+                if attempt < 2:
+                    await asyncio.sleep(1.5 * (attempt + 1))
+                    continue
+                raise last_exc
+            return data
+        except (httpx.TimeoutException, httpx.TransportError) as e:
+            last_exc = e
+            if attempt < 2:
+                await asyncio.sleep(1.5 * (attempt + 1))
+                continue
+            raise
+    raise last_exc  # pragma: no cover — loop always returns or raises
+
+
+async def _hosted_open(payload: dict, timeout: float | None):
+    """Walk the provider chain; return the first success. Providers without a configured key
+    are skipped, so a project with one vendor behaves exactly as before.
+
+    On a failover, probe the provider that failed and log ONE greppable line with a verdict —
+    `provider=healthy` means the fault was ours. Logged, never pushed: a failover that
+    succeeded is not an on-fire event (fleet notification standard)."""
+    usable = [p for p in HOSTED_CHAIN
+              if p in HOSTED_PROVIDERS and os.environ.get(HOSTED_PROVIDERS[p]["key_var"])]
+    if not usable:
+        raise RuntimeError(
+            f"no hosted-open provider configured — set one of: "
+            f"{', '.join(HOSTED_PROVIDERS[p]['key_var'] for p in HOSTED_CHAIN if p in HOSTED_PROVIDERS)}"
+        )
+
+    first_error: Exception | None = None
+    failed_from: str | None = None
+    for provider in usable:
+        try:
+            data = await _hosted_once(provider, payload, timeout)
+        except Exception as e:
+            if first_error is None:
+                first_error, failed_from = e, provider
+            log.warning("hosted-open provider %s failed: %s: %s",
+                        provider, type(e).__name__, e)
+            continue
+
+        if failed_from:
+            health = await _probe_provider(failed_from)
+            verdict = {"healthy": "ours", "down": "theirs"}.get(health, "unknown")
+            log.warning(
+                "ATHENA_FALLBACK fired=t from=%s to=%s model=%s err=%s provider=%s verdict=%s",
+                failed_from, provider, payload.get("model"),
+                type(first_error).__name__, health, verdict,
+            )
+        return data
+
+    raise first_error
+
+
+async def complete(
+    system: str,
+    user: str,
+    *,
+    backend: str,
+    model: str,
+    max_tokens: int = 512,
+    temperature: float = 0.0,
+    json_mode: bool = False,
+    images: list | None = None,
+    timeout: float | None = None,
+    num_ctx: int | None = None,
+    eval: bool = False,
+    return_usage: bool = False,
+    api_key: str | None = None,
+):
+    """Return the model's text completion.
+
+    backend: "anthropic" (frontier), "ollama" (local, self-hosted), or "deepinfra" (hosted
+        open — OpenAI-compatible). NOTE: "deepinfra" now names the hosted-open TIER, not a
+        single vendor: it starts at DeepInfra and fails over along ATHENA_HOSTED_CHAIN. Parse
+        hosted-open JSON with `loads_lenient`, not raw `json.loads`.
+    json_mode: forces valid JSON output on the local (Ollama `format:"json"`) and hosted paths,
+        eliminating the sloppy-JSON failure mode of open models. Ignored on the Anthropic path
+        (callers already prompt for JSON there).
+    temperature: applied to local and hosted paths; the Anthropic path omits it so newer models
+        that reject sampling params don't 400.
+    images: optional list of images (paths, raw bytes, or (bytes|path, media_type) pairs) for a
+        vision task — e.g. rasterized PDF pages. Works on all three backends, so the same images
+        can be A/B'd through a local VLM, a hosted VLM, and the frontier model. Requires a
+        vision-capable model.
+    num_ctx: local path only — Ollama's context window in tokens. MUST be set for long-input
+        tasks: Ollama otherwise defaults to ~2048 and SILENTLY truncates the prompt to the tail.
+    eval: frontier arm of an A/B — route Anthropic through the dedicated ANTHROPIC_EVAL_API_KEY
+        (hard-fails if unset), so an eval can never bill to or trip the live product's cap.
+    return_usage: when True, return ``(text, {"input": int, "output": int})`` instead of just
+        text — so a caller can meter the actual token usage. Zeros if the backend omits it.
+    """
+    if backend == "anthropic":
+        if images:
+            content: list = [
+                {"type": "image",
+                 "source": {"type": "base64", "media_type": mt, "data": b64}}
+                for b64, mt in (_encode_image(i) for i in images)
+            ]
+            content.append({"type": "text", "text": user})
+        else:
+            content = user
+        # `eval=True` is opt-in, so a harness that simply forgets it bills the PRODUCT's key and
+        # nothing complains (that is exactly how a packet-QC A/B billed BidScout's prod key on
+        # 2026-08-01). Athena's tooling sets ATHENA_EVAL_MODE=1; under it, an un-flagged frontier
+        # call is a bug, so fail loudly instead of spending. Product code never sets the var.
+        if os.environ.get("ATHENA_EVAL_MODE") == "1" and not eval:
+            raise RuntimeError(
+                "ATHENA_EVAL_MODE=1 but complete(backend='anthropic') was called without eval=True — "
+                "that would bill the PRODUCT's ANTHROPIC_API_KEY for an eval. Pass eval=True to route "
+                "through ANTHROPIC_EVAL_API_KEY."
+            )
+        # An explicit api_key wins over the environment. Why this exists: a caller can hold a
+        # perfectly good key in CONFIG (pydantic reading a .env FILE) while os.environ has
+        # none — a systemd unit without EnvironmentFile, a cron job, a bare script. The guard
+        # `if not settings.anthropic_api_key` then PASSES and this client still fails auth,
+        # which reads as coverage while providing none. That divergence caused three separate
+        # QuantumPools failures on 2026-08-04 alone, one of them a regulatory read that had
+        # never once succeeded. Callers holding a key should pass it rather than hope the
+        # process inherited it. Hosted-open providers still resolve from the environment:
+        # the chain fails over ACROSS vendors, so a single key cannot serve them.
+        if api_key and not eval:
+            client = anthropic.AsyncAnthropic(api_key=api_key)
+        else:
+            client = eval_anthropic_client() if eval else _anthropic()
+        resp = await client.messages.create(
+            model=model,
+            max_tokens=max_tokens,
+            system=system,
+            messages=[{"role": "user", "content": content}],
+        )
+        text = resp.content[0].text
+        if return_usage:
+            u = getattr(resp, "usage", None)
+            return text, {"input": getattr(u, "input_tokens", 0) or 0,
+                          "output": getattr(u, "output_tokens", 0) or 0}
+        return text
+
+    if backend == "ollama":
+        user_msg: dict = {"role": "user", "content": user}
+        if images:
+            user_msg["images"] = [b64 for b64, _ in (_encode_image(i) for i in images)]
+        payload: dict = {
+            "model": model,
+            "messages": [{"role": "system", "content": system}, user_msg],
+            "stream": False,
+            "options": {"num_predict": max_tokens, "temperature": temperature},
+        }
+        if num_ctx is not None:
+            payload["options"]["num_ctx"] = num_ctx
+        if json_mode:
+            payload["format"] = "json"
+        async with httpx.AsyncClient(timeout=timeout or DEFAULT_TIMEOUT) as client:
+            r = await client.post(f"{OLLAMA_URL}/api/chat", json=payload)
+            r.raise_for_status()
+            data = r.json()
+            if return_usage:
+                return data["message"]["content"], {
+                    "input": data.get("prompt_eval_count", 0) or 0,
+                    "output": data.get("eval_count", 0) or 0}
+            return data["message"]["content"]
+
+    if backend in ("deepinfra", "hosted", "together"):
+        sys_prompt = system
+        # Qwen3 runs a hidden reasoning pass by default; disable it for fast, deterministic
+        # instruct-style scoring/summarizing (fleet finding — score with /no_think).
+        if "qwen3" in model.lower():
+            sys_prompt = "/no_think\n" + system
+        if images:
+            user_content: list | str = [
+                {"type": "image_url",
+                 "image_url": {"url": f"data:{mt};base64,{b64}"}}
+                for b64, mt in (_encode_image(i) for i in images)
+            ]
+            user_content.append({"type": "text", "text": user})
+        else:
+            user_content = user
+        payload = {
+            "model": model,
+            "messages": [{"role": "system", "content": sys_prompt},
+                         {"role": "user", "content": user_content}],
+            "max_tokens": max_tokens,
+            "temperature": temperature,
+        }
+        if json_mode:
+            payload["response_format"] = {"type": "json_object"}
+
+        # An explicit `backend="together"` pins that vendor; "deepinfra"/"hosted" walk the chain.
+        global HOSTED_CHAIN
+        if backend == "together":
+            data = await _hosted_once("together", payload, timeout)
+        else:
+            data = await _hosted_open(payload, timeout)
+
+        text = data["choices"][0]["message"]["content"]
+        if return_usage:
+            usage = data.get("usage") or {}
+            return text, {"input": usage.get("prompt_tokens", 0) or 0,
+                          "output": usage.get("completion_tokens", 0) or 0}
+        return text
+
+    raise ValueError(
+        f"unknown backend: {backend!r} (expected 'anthropic', 'ollama', or 'deepinfra')")

athena_llm-0.2.0/pyproject.toml

@@ -0,0 +1,22 @@
+# Packaged so projects INSTALL the client instead of copying it.
+#
+# Vendoring drifted in exactly the way you'd predict: BidScout's copy grew a whole
+# hosted-open backend, retries and usage metering that the canonical file never had,
+# so the "shared" client was shared in name only and a fix landed in one place.
+# Pin a tag per app; upgrade is a bump plus a redeploy.
+#
+# Deps stay anthropic + httpx on purpose — a framework was rejected on measurement
+# (27 transitive packages for ~30 lines of retry logic). See docs/provider-redundancy.md.
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "athena-llm"
+version = "0.2.0"
+description = "Athena LLM client — one seam across frontier, hosted-open (with failover), and local backends"
+requires-python = ">=3.10"
+dependencies = ["anthropic>=0.40.0", "httpx>=0.27"]
+
+[tool.setuptools]
+py-modules = ["athena_llm"]

athena_llm-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+