flexinference 0.1.0__tar.gz

flexinference-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+node_modules/
+dist/
+build/
+*.egg-info/
+__pycache__/
+.venv/
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.env*
+*.log
+.DS_Store

flexinference-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Aditya Perswal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

flexinference-0.1.0/PKG-INFO

@@ -0,0 +1,120 @@
+Metadata-Version: 2.4
+Name: flexinference
+Version: 0.1.0
+Summary: Official Python SDK for FlexInference - a deadline-aware, OpenAI-compatible inference router.
+Project-URL: Homepage, https://flexinference.com
+Project-URL: Documentation, https://flexinference.com/docs
+Author-email: Aditya Perswal <adityaperswal@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: ai,flexinference,gpt,inference,llm,openai
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: httpx>=0.27
+Requires-Dist: typing-extensions>=4.12
+Provides-Extra: dev
+Requires-Dist: mypy>=1.13; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.7; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# FlexInference (Python)
+
+The official Python SDK for [FlexInference](https://flexinference.com) - a deadline-aware, OpenAI-compatible inference router. Send the OpenAI requests you already send, bring your own OpenAI key, and add one field - `start_within` - to trade latency for cost.
+
+```bash
+pip install flexinference
+```
+
+## Quickstart
+
+```python
+from flexinference import FlexInference
+
+client = FlexInference(api_key="flex_live_...")
+
+res = client.responses.create({
+    "model": "gpt-5.5",
+    "input": "Write a haiku about cheap GPUs.",
+    "start_within": "00h-00m-30s",
+})
+
+print(res["output_text"])
+```
+
+`start_within` takes `"priority"`, `"standard"`, or a duration `"HHh-MMm-SSs"` (5s-10m) that races OpenAI's flex tier and falls back to standard if it can't start in time. See the [docs](https://flexinference.com/docs/deadline-routing).
+
+## Streaming
+
+```python
+stream = client.responses.create(
+    {"model": "gpt-5-nano", "input": "Count to ten.", "start_within": "00h-00m-20s"},
+    stream=True,
+)
+for event in stream:
+    if event.get("type") == "response.output_text.delta":
+        print(event["delta"], end="")
+```
+
+## Chat Completions
+
+```python
+res = client.chat.completions.create({
+    "model": "gpt-5.5",
+    "messages": [{"role": "user", "content": "Hello!"}],
+    "start_within": "standard",
+})
+print(res["choices"][0]["message"]["content"])
+```
+
+## Closing the client
+
+The client holds a pooled `httpx.Client`, so close it when you're done to release connections. Use it as a context manager:
+
+```python
+with FlexInference(api_key="flex_live_...") as client:
+    res = client.responses.create({"model": "gpt-5.5", "input": "Hi."})
+    print(res["output_text"])
+# connections are released on exit
+```
+
+Or close it yourself:
+
+```python
+client = FlexInference(api_key="flex_live_...")
+try:
+    ...
+finally:
+    client.close()
+```
+
+## Errors
+
+Non-2xx responses raise `FlexInferenceError`, carrying the OpenAI-shaped `status`, `type`, `code`, and `param`:
+
+```python
+from flexinference import FlexInferenceError
+
+try:
+    client.responses.create({"model": "gpt-5.5", "input": "hi", "start_within": "priority"})
+except FlexInferenceError as err:
+    if err.code == "no_byok_key":
+        print("Add your OpenAI key in the dashboard.")
+    else:
+        raise
+```
+
+## Configuration
+
+| Argument | Default | Description |
+| --- | --- | --- |
+| `api_key` | (required) | Your `flex_live_` key. |
+| `base_url` | `https://api.flexinference.com/v1` | Override the router endpoint. |
+| `client` | `httpx.Client` with a 600s read timeout | Provide your own `httpx.Client`. |
+
+## License
+
+MIT

flexinference-0.1.0/README.md

@@ -0,0 +1,97 @@
+# FlexInference (Python)
+
+The official Python SDK for [FlexInference](https://flexinference.com) - a deadline-aware, OpenAI-compatible inference router. Send the OpenAI requests you already send, bring your own OpenAI key, and add one field - `start_within` - to trade latency for cost.
+
+```bash
+pip install flexinference
+```
+
+## Quickstart
+
+```python
+from flexinference import FlexInference
+
+client = FlexInference(api_key="flex_live_...")
+
+res = client.responses.create({
+    "model": "gpt-5.5",
+    "input": "Write a haiku about cheap GPUs.",
+    "start_within": "00h-00m-30s",
+})
+
+print(res["output_text"])
+```
+
+`start_within` takes `"priority"`, `"standard"`, or a duration `"HHh-MMm-SSs"` (5s-10m) that races OpenAI's flex tier and falls back to standard if it can't start in time. See the [docs](https://flexinference.com/docs/deadline-routing).
+
+## Streaming
+
+```python
+stream = client.responses.create(
+    {"model": "gpt-5-nano", "input": "Count to ten.", "start_within": "00h-00m-20s"},
+    stream=True,
+)
+for event in stream:
+    if event.get("type") == "response.output_text.delta":
+        print(event["delta"], end="")
+```
+
+## Chat Completions
+
+```python
+res = client.chat.completions.create({
+    "model": "gpt-5.5",
+    "messages": [{"role": "user", "content": "Hello!"}],
+    "start_within": "standard",
+})
+print(res["choices"][0]["message"]["content"])
+```
+
+## Closing the client
+
+The client holds a pooled `httpx.Client`, so close it when you're done to release connections. Use it as a context manager:
+
+```python
+with FlexInference(api_key="flex_live_...") as client:
+    res = client.responses.create({"model": "gpt-5.5", "input": "Hi."})
+    print(res["output_text"])
+# connections are released on exit
+```
+
+Or close it yourself:
+
+```python
+client = FlexInference(api_key="flex_live_...")
+try:
+    ...
+finally:
+    client.close()
+```
+
+## Errors
+
+Non-2xx responses raise `FlexInferenceError`, carrying the OpenAI-shaped `status`, `type`, `code`, and `param`:
+
+```python
+from flexinference import FlexInferenceError
+
+try:
+    client.responses.create({"model": "gpt-5.5", "input": "hi", "start_within": "priority"})
+except FlexInferenceError as err:
+    if err.code == "no_byok_key":
+        print("Add your OpenAI key in the dashboard.")
+    else:
+        raise
+```
+
+## Configuration
+
+| Argument | Default | Description |
+| --- | --- | --- |
+| `api_key` | (required) | Your `flex_live_` key. |
+| `base_url` | `https://api.flexinference.com/v1` | Override the router endpoint. |
+| `client` | `httpx.Client` with a 600s read timeout | Provide your own `httpx.Client`. |
+
+## License
+
+MIT

flexinference-0.1.0/pyproject.toml

@@ -0,0 +1,79 @@
+[project]
+name = "flexinference"
+version = "0.1.0"
+description = "Official Python SDK for FlexInference - a deadline-aware, OpenAI-compatible inference router."
+readme = "README.md"
+license = { text = "MIT" }
+authors = [{ name = "Aditya Perswal", email = "adityaperswal@gmail.com" }]
+requires-python = ">=3.12"
+keywords = ["flexinference", "openai", "llm", "inference", "ai", "gpt"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Typing :: Typed",
+]
+dependencies = [
+    "httpx>=0.27",
+    "typing-extensions>=4.12",
+]
+
+[project.urls]
+Homepage = "https://flexinference.com"
+Documentation = "https://flexinference.com/docs"
+
+[project.optional-dependencies]
+dev = [
+    "mypy>=1.13",
+    "ruff>=0.7",
+    "pytest>=8",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/flexinference"]
+
+[tool.mypy]
+strict = true
+python_version = "3.12"
+
+# models.py is generated by datamodel-code-generator; its types are used to check
+# our code, but its own TypedDict-extension quirks are not ours to fix.
+[[tool.mypy.overrides]]
+module = "flexinference.models"
+ignore_errors = true
+
+[tool.ruff]
+line-length = 100
+extend-exclude = ["src/flexinference/models.py"]
+
+[tool.ruff.lint]
+# A strong, library-appropriate rule set for a typed HTTP client. No ANN: mypy
+# strict already enforces annotations, so ANN would be redundant noise. No app
+# rules (FastAPI/SQLAlchemy/web) - this is just a typed client.
+select = [
+    "E",   # pycodestyle errors
+    "F",   # pyflakes
+    "W",   # pycodestyle warnings
+    "I",   # isort
+    "UP",  # pyupgrade
+    "B",   # flake8-bugbear
+    "SIM", # flake8-simplify
+    "N",   # pep8-naming
+    "C4",  # flake8-comprehensions
+    "PT",  # flake8-pytest-style
+    "RUF", # ruff-specific rules
+    "PIE", # flake8-pie
+    "TID", # flake8-tidy-imports
+]
+
+[tool.ruff.lint.per-file-ignores]
+# Tests assert freely and may carry long literal SSE/fixture lines.
+"tests/**" = ["S101", "E501"]
+# models.py is generated by datamodel-code-generator.
+"src/flexinference/models.py" = ["ALL"]
+
+[tool.ruff.format]

flexinference-0.1.0/src/flexinference/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from . import models
+from ._client import DEFAULT_BASE_URL, FlexInference, FlexInferenceError
+
+__all__ = ["DEFAULT_BASE_URL", "FlexInference", "FlexInferenceError", "models"]

flexinference-0.1.0/src/flexinference/_client.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator, Mapping
+from typing import Any, Literal, cast, overload
+
+import httpx
+
+from .models import (
+    CreateChatCompletionRequest,
+    CreateChatCompletionResponse,
+    CreateChatCompletionStreamResponse,
+    CreateResponse,
+    Response,
+    ResponseStreamEvent,
+)
+
+DEFAULT_BASE_URL = "https://api.flexinference.com/v1"
+
+
+class FlexInferenceError(Exception):
+    """An error returned by FlexInference or by the upstream provider, passed through."""
+
+    def __init__(self, status: int, body: Mapping[str, Any] | None, fallback: str) -> None:
+        err = body.get("error") if isinstance(body, Mapping) else None
+        err = err if isinstance(err, Mapping) else {}
+        super().__init__(err.get("message") or fallback)
+        self.status = status
+        self.type: str | None = err.get("type")
+        self.code: str | None = err.get("code")
+        self.param: str | None = err.get("param")
+
+
+class FlexInference:
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str = DEFAULT_BASE_URL,
+        client: httpx.Client | None = None,
+    ) -> None:
+        if not api_key:
+            raise ValueError("FlexInference: `api_key` is required.")
+        self._base_url = base_url.rstrip("/")
+        # Inference calls routinely run far longer than httpx's 5s default; match the
+        # OpenAI SDK's generous read budget while keeping a short connect timeout.
+        self._client = client or httpx.Client(timeout=httpx.Timeout(600.0, connect=10.0))
+        self._headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+            "Accept": "text/event-stream, application/json",
+        }
+        self.responses = _Responses(self)
+        self.chat = _Chat(self)
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> FlexInference:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def _post_json(self, path: str, payload: Mapping[str, Any]) -> Any:
+        r = self._client.post(
+            f"{self._base_url}{path}", headers=self._headers, content=json.dumps(payload)
+        )
+        if r.status_code >= 400:
+            raise FlexInferenceError(r.status_code, _safe_json(r), f"HTTP {r.status_code}")
+        return r.json()
+
+    def _post_stream(self, path: str, payload: Mapping[str, Any]) -> Iterator[Any]:
+        with self._client.stream(
+            "POST", f"{self._base_url}{path}", headers=self._headers, content=json.dumps(payload)
+        ) as r:
+            if r.status_code >= 400:
+                r.read()
+                raise FlexInferenceError(r.status_code, _safe_json(r), f"HTTP {r.status_code}")
+            yield from _parse_sse(r)
+
+
+def _safe_json(r: httpx.Response) -> Mapping[str, Any] | None:
+    try:
+        data = r.json()
+    except (json.JSONDecodeError, ValueError):
+        return None
+    return data if isinstance(data, Mapping) else None
+
+
+def _parse_sse(r: httpx.Response) -> Iterator[Any]:
+    data_lines: list[str] = []
+    for line in r.iter_lines():
+        if line == "":
+            if data_lines:
+                data = "\n".join(data_lines)
+                data_lines = []
+                if data == "[DONE]":
+                    return
+                yield json.loads(data)
+            continue
+        if line.startswith("data:"):
+            data_lines.append(line[5:].lstrip(" "))
+    if data_lines:
+        data = "\n".join(data_lines)
+        if data != "[DONE]":
+            yield json.loads(data)
+
+
+class _Responses:
+    def __init__(self, parent: FlexInference) -> None:
+        self._parent = parent
+
+    @overload
+    def create(self, body: CreateResponse, *, stream: Literal[False] = False) -> Response: ...
+    @overload
+    def create(
+        self, body: CreateResponse, *, stream: Literal[True]
+    ) -> Iterator[ResponseStreamEvent]: ...
+    def create(
+        self, body: CreateResponse, *, stream: bool = False
+    ) -> Response | Iterator[ResponseStreamEvent]:
+        payload = dict(body)
+        payload["stream"] = stream
+        if stream:
+            return self._parent._post_stream("/responses", payload)
+        return cast(Response, self._parent._post_json("/responses", payload))
+
+
+class _Chat:
+    def __init__(self, parent: FlexInference) -> None:
+        self.completions = _ChatCompletions(parent)
+
+
+class _ChatCompletions:
+    def __init__(self, parent: FlexInference) -> None:
+        self._parent = parent
+
+    @overload
+    def create(
+        self, body: CreateChatCompletionRequest, *, stream: Literal[False] = False
+    ) -> CreateChatCompletionResponse: ...
+    @overload
+    def create(
+        self, body: CreateChatCompletionRequest, *, stream: Literal[True]
+    ) -> Iterator[CreateChatCompletionStreamResponse]: ...
+    def create(
+        self, body: CreateChatCompletionRequest, *, stream: bool = False
+    ) -> CreateChatCompletionResponse | Iterator[CreateChatCompletionStreamResponse]:
+        payload = dict(body)
+        payload["stream"] = stream
+        if stream:
+            return self._parent._post_stream("/chat/completions", payload)
+        return cast(
+            CreateChatCompletionResponse, self._parent._post_json("/chat/completions", payload)
+        )