llmf 0.1.0__py3-none-any.whl

llmf/__init__.py

@@ -0,0 +1,87 @@
+"""llmf (llmfree) - one simple call for free, keyless LLM endpoints.
+
+    >>> import llmf
+    >>> llmf.chat("Xin chào!")            # no key, no setup, automatic fallback
+    >>> llmf.chat("Explain GIL", model="best")
+    >>> for piece in llmf.stream("Count to 5"): print(piece, end="")
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+from typing import Any, List, Optional
+
+from ._version import __version__
+from .client import Client, Session, Stream
+from .errors import AllProvidersFailed, LLMFError, ModelNotFound, ProviderError
+from ._types import Attempt, HealthResult, ModelInfo, ProviderInfo, Reply
+
+_lock = threading.Lock()
+_default: Optional[Client] = None
+
+
+def default_client() -> Client:
+    """The shared client behind the module-level functions."""
+    global _default
+    with _lock:
+        if _default is None:
+            kw: dict = {}
+            if os.environ.get("LLMF_TIMEOUT"):
+                kw["timeout"] = float(os.environ["LLMF_TIMEOUT"])
+            if os.environ.get("LLMF_MODEL"):
+                kw["model"] = os.environ["LLMF_MODEL"]
+            _default = Client(**kw)
+        return _default
+
+
+def chat(prompt, **kw: Any) -> Reply:
+    return default_client().chat(prompt, **kw)
+
+
+def stream(prompt, **kw: Any) -> Stream:
+    return default_client().stream(prompt, **kw)
+
+
+async def achat(prompt, **kw: Any) -> Reply:
+    return await default_client().achat(prompt, **kw)
+
+
+def astream(prompt, **kw: Any):
+    return default_client().astream(prompt, **kw)
+
+
+def session(system: Optional[str] = None, **defaults: Any) -> Session:
+    return default_client().session(system, **defaults)
+
+
+def models(live: bool = False) -> List[ModelInfo]:
+    return default_client().models(live=live)
+
+
+def providers() -> List[ProviderInfo]:
+    return default_client().providers()
+
+
+def health() -> List[HealthResult]:
+    return default_client().health()
+
+
+def register(*a: Any, **kw: Any):
+    return default_client().register(*a, **kw)
+
+
+def discover(**kw: Any) -> List[str]:
+    return default_client().discover(**kw)
+
+
+def configure(**kw: Any) -> Client:
+    return default_client().configure(**kw)
+
+
+__all__ = [
+    "__version__", "Client", "Session", "Stream", "Reply", "chat", "stream", "achat", "astream",
+    "session", "models", "providers", "health", "register", "discover", "configure",
+    "default_client", "LLMFError", "ProviderError", "AllProvidersFailed", "ModelNotFound",
+    "Attempt", "HealthResult", "ModelInfo", "ProviderInfo",
+]

llmf/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

llmf/_http.py

@@ -0,0 +1,67 @@
+"""Tiny stdlib HTTP layer (no third-party dependencies)."""
+
+from __future__ import annotations
+
+import gzip
+import http.client
+import socket
+import ssl
+import urllib.error
+import urllib.request
+from typing import Dict, Optional
+
+from ._version import __version__
+
+DEFAULT_UA = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/130.0.0.0 Safari/537.36 llmf/%s" % __version__
+)
+
+NETWORK_ERRORS = (OSError, http.client.HTTPException, ssl.SSLError, socket.timeout)
+
+
+class HTTPFailure(Exception):
+    """Transport-level failure. ``status is None`` means no HTTP response."""
+
+    def __init__(self, status: Optional[int], body: str = "", kind: str = "http") -> None:
+        self.status = status
+        self.body = body
+        self.kind = kind  # "http" | "timeout" | "network"
+        super().__init__("%s %s" % (status if status is not None else kind, body[:120]))
+
+
+def decode_body(resp) -> str:
+    data = resp.read()
+    if resp.headers.get("Content-Encoding", "").lower() == "gzip":
+        try:
+            data = gzip.decompress(data)
+        except OSError:
+            pass
+    return data.decode("utf-8", "replace")
+
+
+def open_url(
+    url: str,
+    *,
+    headers: Dict[str, str],
+    data: Optional[bytes] = None,
+    timeout: float = 60.0,
+):
+    """Open ``url`` and return the response object (caller must close it)."""
+    req = urllib.request.Request(url, data=data, headers=headers)
+    try:
+        return urllib.request.urlopen(req, timeout=timeout)
+    except urllib.error.HTTPError as e:
+        try:
+            body = decode_body(e)
+        except Exception:  # pragma: no cover - body unreadable
+            body = ""
+        raise HTTPFailure(e.code, body) from None
+    except urllib.error.URLError as e:
+        reason = e.reason
+        kind = "timeout" if isinstance(reason, (socket.timeout, TimeoutError)) else "network"
+        raise HTTPFailure(None, str(reason), kind) from None
+    except (socket.timeout, TimeoutError) as e:
+        raise HTTPFailure(None, str(e) or "timed out", "timeout") from None
+    except NETWORK_ERRORS as e:
+        raise HTTPFailure(None, str(e), "network") from None

llmf/_messages.py

@@ -0,0 +1,60 @@
+"""Turn friendly inputs (str / messages / images) into OpenAI-style messages."""
+
+from __future__ import annotations
+
+import base64
+import copy
+import mimetypes
+import os
+from typing import Any, List, Optional, Sequence, Union
+
+Prompt = Union[str, Sequence[dict]]
+
+
+def _sniff(data: bytes) -> str:
+    if data.startswith(b"\x89PNG"):
+        return "image/png"
+    if data.startswith(b"\xff\xd8"):
+        return "image/jpeg"
+    if data.startswith(b"GIF8"):
+        return "image/gif"
+    if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
+        return "image/webp"
+    return "application/octet-stream"
+
+
+def image_url(img: Any) -> str:
+    """URL / data-URI / local path / raw bytes -> something an API accepts."""
+    if isinstance(img, (bytes, bytearray)):
+        data = bytes(img)
+        return "data:%s;base64,%s" % (_sniff(data), base64.b64encode(data).decode())
+    s = str(img)
+    if s.startswith(("http://", "https://", "data:")):
+        return s
+    if os.path.isfile(s):
+        with open(s, "rb") as f:
+            data = f.read()
+        mime = mimetypes.guess_type(s)[0] or _sniff(data)
+        return "data:%s;base64,%s" % (mime, base64.b64encode(data).decode())
+    raise FileNotFoundError("image not found: %r" % s)
+
+
+def build_messages(prompt: Prompt, system: Optional[str] = None,
+                   images: Optional[Sequence[Any]] = None) -> List[dict]:
+    if isinstance(prompt, str):
+        msgs: List[dict] = [{"role": "user", "content": prompt}]
+    else:
+        msgs = copy.deepcopy([dict(m) for m in prompt])
+        if not msgs:
+            raise ValueError("messages must not be empty")
+    if images:
+        idx = next((i for i in range(len(msgs) - 1, -1, -1) if msgs[i].get("role") == "user"), None)
+        if idx is None:
+            raise ValueError("images need a user message to attach to")
+        content = msgs[idx].get("content", "")
+        parts = [{"type": "text", "text": content}] if isinstance(content, str) else list(content)
+        parts += [{"type": "image_url", "image_url": {"url": image_url(i)}} for i in images]
+        msgs[idx]["content"] = parts
+    if system and not (msgs and msgs[0].get("role") == "system"):
+        msgs.insert(0, {"role": "system", "content": system})
+    return msgs

llmf/_think.py

@@ -0,0 +1,84 @@
+"""Split ``<think>...</think>`` reasoning out of a (possibly streamed) text."""
+
+from __future__ import annotations
+
+from typing import List
+
+OPEN = "<think>"
+CLOSE = "</think>"
+
+
+def _partial_suffix(s: str, token: str) -> int:
+    """Length of the longest proper prefix of ``token`` that ``s`` ends with."""
+    for k in range(min(len(token) - 1, len(s)), 0, -1):
+        if s.endswith(token[:k]):
+            return k
+    return 0
+
+
+class ThinkSplitter:
+    """Incremental splitter. ``feed()`` returns the visible text for a chunk."""
+
+    def __init__(self) -> None:
+        self._buf = ""
+        self._inside = False
+        self._started = False  # visible text has begun (for leading-space trim)
+        self.thought: List[str] = []
+
+    def _emit(self, text: str) -> str:
+        if not self._started:
+            text = text.lstrip()
+            if text:
+                self._started = True
+        return text
+
+    def feed(self, chunk: str) -> str:
+        self._buf += chunk
+        out: List[str] = []
+        while True:
+            if not self._inside:
+                i = self._buf.find(OPEN)
+                if i >= 0:
+                    out.append(self._buf[:i])
+                    self._buf = self._buf[i + len(OPEN):]
+                    self._inside = True
+                    continue
+                keep = _partial_suffix(self._buf, OPEN)
+                cut = len(self._buf) - keep
+                out.append(self._buf[:cut])
+                self._buf = self._buf[cut:]
+                break
+            j = self._buf.find(CLOSE)
+            if j >= 0:
+                self.thought.append(self._buf[:j])
+                self._buf = self._buf[j + len(CLOSE):]
+                self._inside = False
+                continue
+            keep = _partial_suffix(self._buf, CLOSE)
+            cut = len(self._buf) - keep
+            self.thought.append(self._buf[:cut])
+            self._buf = self._buf[cut:]
+            break
+        return self._emit("".join(out))
+
+    def flush(self) -> str:
+        rest, self._buf = self._buf, ""
+        if self._inside:
+            self.thought.append(rest)
+            return ""
+        return self._emit(rest)
+
+    @property
+    def reasoning(self) -> str:
+        return "".join(self.thought).strip()
+
+
+def split_think(text: str):
+    """Non-streaming helper -> ``(visible_text, reasoning_or_None)``."""
+    # Some templates only emit the closing tag (the opening one is in the prompt).
+    if CLOSE in text and OPEN not in text:
+        thought, _, text = text.partition(CLOSE)
+        return text.strip(), (thought.strip() or None)
+    sp = ThinkSplitter()
+    visible = sp.feed(text) + sp.flush()
+    return visible.strip(), (sp.reasoning or None)

llmf/_types.py

@@ -0,0 +1,114 @@
+"""Public value types."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+
+@dataclass
+class Attempt:
+    """One provider/model try (successful or not)."""
+
+    provider: str
+    model: str
+    ok: bool
+    latency: float = 0.0
+    error: Optional[str] = None
+
+
+class Reply(str):
+    """The answer text, plus metadata.
+
+    A ``Reply`` *is* a ``str`` – print it, slice it, ``json.loads`` it – and
+    additionally carries ``provider``, ``model``, ``reasoning``, ``usage``,
+    ``finish_reason``, ``tool_calls``, ``latency``, ``raw`` and ``attempts``.
+    """
+
+    provider: str
+    model: str
+    reasoning: Optional[str]
+    usage: Optional[Dict[str, Any]]
+    finish_reason: Optional[str]
+    tool_calls: Optional[List[Any]]
+    latency: float
+    raw: Any
+    attempts: List[Attempt]
+
+    def __new__(
+        cls,
+        text: str = "",
+        *,
+        provider: str = "",
+        model: str = "",
+        reasoning: Optional[str] = None,
+        usage: Optional[Dict[str, Any]] = None,
+        finish_reason: Optional[str] = None,
+        tool_calls: Optional[List[Any]] = None,
+        latency: float = 0.0,
+        raw: Any = None,
+        attempts: Optional[List[Attempt]] = None,
+    ) -> "Reply":
+        self = super().__new__(cls, text)
+        self.provider = provider
+        self.model = model
+        self.reasoning = reasoning
+        self.usage = usage
+        self.finish_reason = finish_reason
+        self.tool_calls = tool_calls
+        self.latency = latency
+        self.raw = raw
+        self.attempts = attempts if attempts is not None else []
+        return self
+
+    @property
+    def text(self) -> str:
+        return str.__str__(self)
+
+    @property
+    def source(self) -> str:
+        """``"provider/model"`` that produced this reply."""
+        return "%s/%s" % (self.provider, self.model)
+
+    def __getnewargs_ex__(self):  # keeps copy/pickle working for a str subclass
+        meta = dict(
+            provider=self.provider, model=self.model, reasoning=self.reasoning,
+            usage=self.usage, finish_reason=self.finish_reason,
+            tool_calls=self.tool_calls, latency=self.latency, raw=self.raw,
+            attempts=self.attempts,
+        )
+        return (self.text,), meta
+
+
+@dataclass
+class ModelInfo:
+    provider: str
+    id: str
+    tags: Tuple[str, ...] = ()
+    vision: bool = False
+    verified: bool = True
+
+    @property
+    def name(self) -> str:
+        """Fully-qualified name usable as ``model=``."""
+        return "%s/%s" % (self.provider, self.id)
+
+
+@dataclass
+class ProviderInfo:
+    name: str
+    base_url: str
+    kind: str
+    default_model: str
+    models: List[str] = field(default_factory=list)
+    min_interval: float = 0.0
+    note: str = ""
+
+
+@dataclass
+class HealthResult:
+    provider: str
+    model: str
+    ok: bool
+    latency: float = 0.0
+    error: Optional[str] = None

llmf/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

llmf/cli.py

@@ -0,0 +1,150 @@
+"""Command line: ``llmf "your question"``."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from typing import List, Optional
+
+from . import __version__, default_client
+from .errors import LLMFError
+
+COMMANDS = ("chat", "models", "providers", "health", "version")
+
+HELP = """\
+usage: llmf [options] "question"          ask once (also reads stdin)
+       llmf -i                            interactive chat
+       llmf models | providers | health   inspect what is available
+"""
+
+
+def _parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="llmf", usage=HELP, add_help=True,
+                                description="Free, keyless LLMs behind one command.")
+    p.add_argument("words", nargs="*", help="question, or one of: " + ", ".join(COMMANDS))
+    p.add_argument("-m", "--model", help="model, alias (fast/best/code/vision/reasoning) or provider/model")
+    p.add_argument("-s", "--system", help="system prompt")
+    p.add_argument("-i", "--interactive", action="store_true", help="interactive chat (REPL)")
+    p.add_argument("--image", action="append", default=[], help="image path/URL (repeatable)")
+    p.add_argument("--max-tokens", type=int)
+    p.add_argument("-t", "--temperature", type=float)
+    p.add_argument("--stream", action="store_true", help="print the answer as it arrives")
+    p.add_argument("--json", action="store_true", help="machine-readable output")
+    p.add_argument("--live", action="store_true", help="models: also query providers live")
+    p.add_argument("--timeout", type=float)
+    p.add_argument("-v", "--verbose", action="store_true", help="show provider/model used")
+    p.add_argument("-V", "--version", action="version", version="llmf " + __version__)
+    return p
+
+
+def _read_stdin() -> str:
+    return "" if sys.stdin is None or sys.stdin.isatty() else sys.stdin.read().strip()
+
+
+def _ask(a, prompt, history: Optional[List[dict]] = None) -> str:
+    c = default_client()
+    kw = dict(model=a.model, system=a.system, max_tokens=a.max_tokens, temperature=a.temperature)
+    if a.image:
+        kw["images"] = a.image
+    msgs = (history or []) + [{"role": "user", "content": prompt}]
+    if a.stream and not a.json:
+        st = c.stream(msgs, **kw)
+        for piece in st:
+            sys.stdout.write(piece)
+            sys.stdout.flush()
+        sys.stdout.write("\n")
+        reply = st.reply
+        text = reply.text if reply else st.text
+        src = reply.source if reply else ""
+    else:
+        reply = c.chat(msgs, **kw)
+        text, src = reply.text, reply.source
+        if a.json:
+            print(json.dumps({"text": text, "provider": reply.provider, "model": reply.model,
+                              "reasoning": reply.reasoning, "usage": reply.usage,
+                              "latency": round(reply.latency, 3)}, ensure_ascii=False))
+        else:
+            print(text)
+    if a.verbose and src:
+        print("[%s]" % src, file=sys.stderr)
+    return text
+
+
+def _repl(a) -> int:
+    history: List[dict] = []
+    print("llmf %s - /reset clears history, /exit quits" % __version__, file=sys.stderr)
+    while True:
+        try:
+            line = input("> ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print()
+            return 0
+        if not line:
+            continue
+        if line in ("/exit", "/quit"):
+            return 0
+        if line == "/reset":
+            history.clear()
+            continue
+        try:
+            text = _ask(a, line, history)
+        except LLMFError as e:
+            print("error: %s" % e, file=sys.stderr)
+            continue
+        history += [{"role": "user", "content": line}, {"role": "assistant", "content": text}]
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    a = _parser().parse_args(argv)
+    if a.verbose:
+        logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s")
+    c = default_client()
+    if a.timeout:
+        c.configure(timeout=a.timeout)
+    cmd = a.words[0] if a.words and a.words[0] in COMMANDS and len(a.words) == 1 else None
+    try:
+        if cmd == "version":
+            print(__version__)
+        elif cmd == "providers":
+            rows = c.providers()
+            if a.json:
+                print(json.dumps([r.__dict__ for r in rows], ensure_ascii=False, indent=2))
+            for r in rows if not a.json else []:
+                print("%-16s default=%-42s %s" % (r.name, r.default_model, r.note))
+        elif cmd == "models":
+            rows = c.models(live=a.live)
+            if a.json:
+                print(json.dumps([{"name": m.name, "tags": m.tags, "vision": m.vision,
+                                   "verified": m.verified} for m in rows], indent=2))
+            for m in rows if not a.json else []:
+                tags = list(m.tags) + (["vision"] if m.vision and "vision" not in m.tags else [])
+                flags = ",".join(tags + ([] if m.verified else ["unverified"]))
+                print("%-60s %s" % (m.name, flags))
+        elif cmd == "health":
+            bad = 0
+            for h in c.health():
+                bad += not h.ok
+                print("%-4s %-16s %-42s %5.1fs %s" % ("OK" if h.ok else "FAIL", h.provider,
+                                                     h.model, h.latency, h.error or ""))
+            return 1 if bad == len(c.providers()) else 0
+        elif a.interactive:
+            return _repl(a)
+        else:
+            prompt = " ".join(a.words[1:] if a.words[:1] == ["chat"] else a.words).strip()
+            piped = _read_stdin()
+            prompt = ("%s\n\n%s" % (prompt, piped)).strip() if piped else prompt
+            if not prompt:
+                print(HELP, file=sys.stderr)
+                return 2
+            _ask(a, prompt)
+    except LLMFError as e:
+        print("error: %s" % e, file=sys.stderr)
+        return 1
+    except FileNotFoundError as e:
+        print("error: %s" % e, file=sys.stderr)
+        return 2
+    except BrokenPipeError:  # e.g. ``llmf ... | head``
+        return 0
+    return 0