steadybatch 0.2.1__py3-none-any.whl

steadybatch/__init__.py

@@ -0,0 +1,7 @@
+"""steadybatch: run millions of LLM requests through batch APIs without silent failures."""
+
+from .models import Outcome, Request, Result
+from .runner import RunReport, Runner
+
+__all__ = ["Outcome", "Request", "Result", "RunReport", "Runner"]
+__version__ = "0.2.1"

steadybatch/bench.py

@@ -0,0 +1,317 @@
+"""Benchmark harness: run one workload on one provider and write down what happened.
+
+    steadybatch-bench run --provider openai --model gpt-4o-mini \\
+        --data examples/support_tickets.jsonl --schema examples/support_ticket.schema.json \\
+        --out runs/openai-1 --prices examples/prices.example.json
+
+    steadybatch-bench compare runs/openai-1 runs/openai-2
+
+``run`` writes results.jsonl (one line per record, in input order) and
+summary.json. ``compare`` checks two runs of the same workload against each
+other, which is how we measure reproducibility.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import statistics
+import sys
+from pathlib import Path
+from typing import Any
+
+from .models import Outcome, Request
+from .providers import get_provider
+from .runner import Runner
+
+SYSTEM_PROMPT = (
+    "You read customer support conversations and extract structured facts. "
+    "Answer with a single JSON object and nothing else."
+)
+
+
+def load_records(path: Path, limit: int | None) -> list[dict[str, Any]]:
+    records = []
+    with path.open() as f:
+        for line in f:
+            if line.strip():
+                records.append(json.loads(line))
+                if limit and len(records) >= limit:
+                    break
+    return records
+
+
+def to_requests(records: list[dict[str, Any]], max_tokens: int, instructions: str = "") -> list[Request]:
+    system = SYSTEM_PROMPT + ("\n\n" + instructions.strip() if instructions.strip() else "")
+    return [
+        Request(key=str(r["key"]), system=system, max_tokens=max_tokens, temperature=0.0,
+                messages=[{"role": "user", "content": r["text"]}])
+        for r in records
+    ]
+
+
+def accuracy(records: list[dict[str, Any]], results_by_key: dict[str, Any]) -> dict[str, float]:
+    """Field-level accuracy against the 'labels' in the dataset, when present."""
+    scores: dict[str, list[int]] = {}
+    for r in records:
+        labels = r.get("labels")
+        data = results_by_key.get(str(r["key"]))
+        if not labels or not isinstance(data, dict):
+            continue
+        for field, expected in labels.items():
+            scores.setdefault(field, []).append(int(data.get(field) == expected))
+    return {f: round(sum(v) / len(v), 4) for f, v in scores.items() if v}
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    records = load_records(Path(args.data), args.limit)
+    schema = json.loads(Path(args.schema).read_text()) if args.schema else None
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+
+    runner = Runner(get_provider(args.provider, args.model, thinking=args.thinking), args.model, response_schema=schema,
+                    checkpoint=out / "checkpoint.sqlite", max_attempts=args.max_attempts,
+                    poll_every=args.poll_every, batch_size=args.batch_size,
+                    max_open_batches=args.max_open_batches)
+    instructions = Path(args.instructions).read_text() if args.instructions else ""
+    report = runner.run(to_requests(records, args.max_tokens, instructions))
+
+    with (out / "results.jsonl").open("w") as f:
+        for r in report.results:
+            f.write(json.dumps({"key": r.key, "outcome": r.outcome.value, "data": r.data,
+                                "error": r.error, "attempts": r.attempts, "history": r.history}) + "\n")
+
+    in_tok = sum(r.input_tokens for r in report.results)
+    out_tok = sum(r.output_tokens for r in report.results)
+    summary: dict[str, Any] = {
+        "provider": args.provider,
+        "model": args.model,
+        "instructions_file": args.instructions,
+        "thinking": args.thinking or "model default",
+        "max_tokens": args.max_tokens,
+        "records": len(records),
+        "ok": report.ok,
+        "failed_after_retries": report.failed,
+        "outcomes": {o.value: sum(r.outcome is o for r in report.results) for o in Outcome},
+        "lines_missing_seen": report.lines_missing,
+        "lines_errored_seen": report.lines_errored,
+        "lines_invalid_seen": report.lines_invalid,
+        "batches_submitted": report.batches_submitted,
+        "capacity_refusals": report.capacity_refusals,
+        "submit_retries": report.submit_retries,
+        "retried_records": sum(r.attempts > 1 for r in report.results),
+        "input_tokens": in_tok,
+        "output_tokens": out_tok,
+        "wall_seconds": round(report.seconds, 1),
+    }
+    if report.batch_seconds:
+        summary["batch_seconds_median"] = round(statistics.median(report.batch_seconds), 1)
+        summary["batch_seconds_max"] = round(max(report.batch_seconds), 1)
+    if args.prices:
+        p = json.loads(Path(args.prices).read_text())[args.provider]
+        cost = (in_tok * p["input_per_mtok"] + out_tok * p["output_per_mtok"]) / 1e6 * p.get("batch_multiplier", 1.0)
+        summary["cost_usd"] = round(cost, 4)
+        summary["cost_per_1000_records_usd"] = round(cost / max(len(records), 1) * 1000, 4)
+    acc = accuracy(records, {r.key: r.data for r in report.results})
+    if acc:
+        summary["field_accuracy"] = acc
+    (out / "summary.json").write_text(json.dumps(summary, indent=2))
+    print(json.dumps(summary, indent=2))
+    return 0
+
+
+def cmd_compare(args: argparse.Namespace) -> int:
+    def load(d: str) -> dict[str, Any]:
+        path = Path(d) / "results.jsonl"
+        if not path.exists() and path.with_suffix(".jsonl.gz").exists():
+            import gzip  # large published runs are stored compressed
+            text = gzip.decompress(path.with_suffix(".jsonl.gz").read_bytes()).decode("utf-8")
+        else:
+            text = path.read_text()
+        rows = [json.loads(l) for l in text.splitlines() if l]
+        return {r["key"]: r["data"] for r in rows if r["outcome"] == "ok"}
+
+    a, b = load(args.run_a), load(args.run_b)
+    both = sorted(set(a) & set(b))
+    exact = sum(a[k] == b[k] for k in both)
+    fields: dict[str, list[int]] = {}
+    for k in both:
+        if isinstance(a[k], dict) and isinstance(b[k], dict):
+            for f in set(a[k]) | set(b[k]):
+                fields.setdefault(f, []).append(int(a[k].get(f) == b[k].get(f)))
+    result = {
+        "records_in_both": len(both),
+        "exact_match_rate": round(exact / len(both), 4) if both else None,
+        "field_match_rate": {f: round(sum(v) / len(v), 4) for f, v in sorted(fields.items())},
+    }
+    print(json.dumps(result, indent=2))
+    return 0
+
+
+REPORT_COLUMNS = [
+    ("run", "Run"),
+    ("provider", "Provider"),
+    ("model", "Model"),
+    ("records", "Records"),
+    ("ok", "OK"),
+    ("failed_after_retries", "Failed"),
+    ("lines_missing_seen", "Missing"),
+    ("lines_errored_seen", "Errored"),
+    ("lines_invalid_seen", "Invalid"),
+    ("retried_records", "Retried"),
+    ("batch_seconds_median", "Median batch (s)"),
+    ("batch_seconds_max", "Slowest batch (s)"),
+    ("cost_per_1000_records_usd", "Cost per 1k ($)"),
+    ("mean_field_accuracy", "Accuracy"),
+]
+
+
+def load_summaries(run_dirs: list[str]) -> list[dict[str, Any]]:
+    rows = []
+    for d in run_dirs:
+        path = Path(d) / "summary.json"
+        if not path.exists():
+            print(f"skipping {d}: no summary.json", file=sys.stderr)
+            continue
+        s = json.loads(path.read_text())
+        s["run"] = Path(d).name
+        acc = s.get("field_accuracy") or {}
+        if acc:
+            s["mean_field_accuracy"] = round(sum(acc.values()) / len(acc), 4)
+        rows.append(s)
+    return rows
+
+
+def write_charts(rows: list[dict[str, Any]], out: Path) -> list[str]:
+    try:
+        import matplotlib
+        matplotlib.use("Agg")
+        import matplotlib.pyplot as plt
+    except ImportError:
+        print("matplotlib not installed; skipping charts (pip install matplotlib)", file=sys.stderr)
+        return []
+    labels = [r["run"] for r in rows]
+    made = []
+
+    fig, ax = plt.subplots(figsize=(max(6, len(rows) * 1.2), 4))
+    bottom = [0] * len(rows)
+    for key, name in [("lines_missing_seen", "Missing"), ("lines_errored_seen", "Errored"),
+                      ("lines_invalid_seen", "Invalid JSON")]:
+        vals = [100 * r.get(key, 0) / max(r.get("records", 1), 1) for r in rows]
+        ax.bar(labels, vals, bottom=bottom, label=name)
+        bottom = [b + v for b, v in zip(bottom, vals)]
+    ax.set_ylabel("% of records (before retries)")
+    ax.set_title("Problems caught per run")
+    ax.legend(loc="upper left", bbox_to_anchor=(1.01, 1), frameon=False)
+    fig.tight_layout()
+    fig.savefig(out / "problems.png", dpi=150)
+    plt.close(fig)
+    made.append("problems.png")
+
+    costs = [(r["run"], r["cost_per_1000_records_usd"]) for r in rows if "cost_per_1000_records_usd" in r]
+    if costs:
+        fig, ax = plt.subplots(figsize=(max(6, len(costs) * 1.2), 4))
+        ax.bar([c[0] for c in costs], [c[1] for c in costs])
+        ax.set_ylabel("USD per 1,000 records (incl. retries)")
+        ax.set_title("Cost per 1,000 records")
+        fig.tight_layout()
+        fig.savefig(out / "cost.png", dpi=150)
+        plt.close(fig)
+        made.append("cost.png")
+    return made
+
+
+def add_costs(rows: list[dict[str, Any]], prices_path: str) -> None:
+    """Fill in cost for runs whose summary has token counts but no cost yet."""
+    prices = json.loads(Path(prices_path).read_text())
+    for r in rows:
+        p = prices.get(r.get("provider", ""))
+        if not p or "cost_usd" in r:
+            continue
+        if p.get("model") and p["model"] != r.get("model"):
+            continue  # prices were recorded for a different model
+        cost = (r.get("input_tokens", 0) * p["input_per_mtok"] + r.get("output_tokens", 0) * p["output_per_mtok"]) / 1e6 * p.get("batch_multiplier", 1.0)
+        r["cost_usd"] = round(cost, 6)
+        r["cost_per_1000_records_usd"] = round(cost / max(r.get("records", 1), 1) * 1000, 4)
+
+
+def cmd_report(args: argparse.Namespace) -> int:
+    rows = load_summaries(args.runs)
+    if not rows:
+        print("no runs with a summary.json found", file=sys.stderr)
+        return 1
+    if args.prices:
+        add_costs(rows, args.prices)
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+
+    def cell(row: dict[str, Any], key: str) -> str:
+        v = row.get(key)
+        return "" if v is None else str(v)
+
+    header = [name for _, name in REPORT_COLUMNS]
+    lines = ["| " + " | ".join(header) + " |", "|" + "---|" * len(header)]
+    for r in rows:
+        lines.append("| " + " | ".join(cell(r, k) for k, _ in REPORT_COLUMNS) + " |")
+
+    import csv
+    with (out / "summary.csv").open("w", newline="") as f:
+        w = csv.writer(f)
+        w.writerow(header)
+        for r in rows:
+            w.writerow([cell(r, k) for k, _ in REPORT_COLUMNS])
+
+    charts = [] if args.no_charts else write_charts(rows, out)
+    md = ["# Benchmark results", "",
+          "Counts of missing, errored and invalid lines are before retries; "
+          "OK and Failed are after retries.", "", *lines, ""]
+    md += [f"![{c}]({c})" for c in charts]
+    (out / "summary.md").write_text("\n".join(md) + "\n")
+    print("\n".join(lines))
+    print(f"\nwrote {out / 'summary.md'} and {out / 'summary.csv'}" + (f" and {len(charts)} charts" if charts else ""))
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="steadybatch-bench")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    run = sub.add_parser("run", help="run one workload on one provider")
+    run.add_argument("--provider", required=True, choices=["fake", "openai", "anthropic", "vllm", "gemini", "bedrock"])
+    run.add_argument("--model", required=True)
+    run.add_argument("--data", required=True)
+    run.add_argument("--schema")
+    run.add_argument("--instructions", help="text file with field definitions, added to the system prompt")
+    run.add_argument("--out", required=True)
+    run.add_argument("--limit", type=int)
+    run.add_argument("--max-tokens", type=int, default=512)
+    run.add_argument("--max-attempts", type=int, default=3)
+    run.add_argument("--batch-size", type=int, help="at most this many requests per batch")
+    run.add_argument("--max-open-batches", type=int,
+                     help="wait for open batches to finish before submitting more than this many")
+    run.add_argument("--thinking", choices=["disabled", "adaptive"],
+                     help="Anthropic only: set thinking explicitly instead of using the model default")
+    run.add_argument("--poll-every", type=float, default=60.0)
+    run.add_argument("--prices", help="JSON file with per-provider token prices")
+    run.set_defaults(func=cmd_run)
+
+    cmp_ = sub.add_parser("compare", help="compare two runs of the same workload")
+    cmp_.add_argument("run_a")
+    cmp_.add_argument("run_b")
+    cmp_.set_defaults(func=cmd_compare)
+
+    rep = sub.add_parser("report", help="turn run summaries into a table, CSV and charts")
+    rep.add_argument("runs", nargs="+", help="run folders that contain summary.json")
+    rep.add_argument("--out", default="results/latest")
+    rep.add_argument("--no-charts", action="store_true")
+    rep.add_argument("--prices", help="JSON price file; fills in cost for runs that lack it")
+    rep.set_defaults(func=cmd_report)
+
+    args = parser.parse_args(argv)
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

steadybatch/chunking.py

@@ -0,0 +1,39 @@
+"""Split work into batches that stay under a provider's limits.
+
+Providers cap a batch by request count and by file size. Going over the
+size cap does not always produce a clear error, so we stay under both,
+with a safety margin on size.
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Iterable, Iterator, TypeVar
+
+T = TypeVar("T")
+
+
+def chunk(
+    items: Iterable[T],
+    max_count: int,
+    max_bytes: int,
+    size_of: Callable[[T], int],
+    margin: float = 0.9,
+) -> Iterator[list[T]]:
+    if max_count < 1 or max_bytes < 1:
+        raise ValueError("limits must be positive")
+    byte_budget = int(max_bytes * margin)
+    batch: list[T] = []
+    used = 0
+    for item in items:
+        size = size_of(item)
+        if size > byte_budget:
+            raise ValueError(
+                f"a single request is {size} bytes, over the per-batch budget of {byte_budget}"
+            )
+        if batch and (len(batch) >= max_count or used + size > byte_budget):
+            yield batch
+            batch, used = [], 0
+        batch.append(item)
+        used += size
+    if batch:
+        yield batch

steadybatch/ids.py

@@ -0,0 +1,32 @@
+"""Stable request IDs.
+
+The same record with the same settings always gets the same ID. That is
+what makes retries safe: if a line comes back twice, or you resubmit work
+after a crash, you can tell it is the same request and not count it twice.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from .models import Request
+
+# Anthropic allows [a-zA-Z0-9_-]{1,64}; OpenAI is looser. Stay within both.
+_PREFIX = "sb-"
+_HASH_CHARS = 32
+
+
+def custom_id_for(request: Request, model: str, response_schema: dict[str, Any] | None = None) -> str:
+    payload = {
+        "key": request.key,
+        "model": model,
+        "messages": request.messages,
+        "system": request.system,
+        "max_tokens": request.max_tokens,
+        "temperature": request.temperature,
+        "schema": response_schema,
+    }
+    blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return _PREFIX + hashlib.sha256(blob.encode("utf-8")).hexdigest()[:_HASH_CHARS]

steadybatch/models.py

@@ -0,0 +1,85 @@
+"""Plain data types shared by the runner and the providers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Request:
+    """One unit of work: a record you want the model to process.
+
+    ``key`` is your own identifier for the record (a row id, a file name).
+    It must be unique within a job; steadybatch uses it to put results
+    back in order and to make retries safe.
+    """
+
+    key: str
+    messages: list[dict[str, str]]
+    system: str | None = None
+    max_tokens: int = 1024
+    temperature: float = 0.0
+
+
+@dataclass(frozen=True)
+class PreparedRequest:
+    """A Request plus everything a provider needs to send it."""
+
+    custom_id: str
+    request: Request
+    model: str
+    response_schema: dict[str, Any] | None = None
+
+
+class BatchState(str, Enum):
+    PENDING = "pending"      # accepted, not started
+    RUNNING = "running"
+    DONE = "done"            # provider says it finished (results may still be incomplete)
+    FAILED = "failed"        # the whole batch failed or expired
+
+
+@dataclass
+class BatchStatus:
+    state: BatchState
+    total: int = 0
+    succeeded: int = 0
+    failed: int = 0
+    detail: str = ""
+    # True when the provider refused the work because the account's queue is full,
+    # not because anything is wrong with the requests.
+    over_capacity: bool = False
+
+
+@dataclass
+class RawResult:
+    """What a provider returned for one line, before any checking."""
+
+    custom_id: str
+    ok: bool
+    text: str | None = None
+    error: str | None = None
+    input_tokens: int = 0
+    output_tokens: int = 0
+
+
+class Outcome(str, Enum):
+    OK = "ok"
+    ERROR = "error"          # the provider returned an error for this line
+    MISSING = "missing"      # the provider never returned this line at all
+    INVALID = "invalid"      # a response came back but failed the schema check
+
+
+@dataclass
+class Result:
+    key: str
+    custom_id: str
+    outcome: Outcome
+    data: Any = None
+    text: str | None = None
+    error: str | None = None
+    attempts: int = 0
+    input_tokens: int = 0
+    output_tokens: int = 0
+    history: list[str] = field(default_factory=list)

steadybatch/providers/__init__.py

@@ -0,0 +1,35 @@
+"""Batch providers. Each one is imported lazily so you only need the SDKs you use."""
+
+from __future__ import annotations
+
+from .base import BatchProvider
+from .fake import FakeProvider, demo_answer
+
+
+def get_provider(name: str, model: str | None = None, *, thinking: str | None = None) -> BatchProvider:
+    if name == "fake":
+        # Misbehaves like a real API: a few lines dropped, errored or malformed.
+        return FakeProvider(answer=demo_answer, drop_rate=0.01, error_rate=0.01, invalid_rate=0.01,
+                            fail_first_attempt_only=True)
+    if name == "openai":
+        from .openai_batch import OpenAIBatch
+        return OpenAIBatch()
+    if name == "anthropic":
+        from .anthropic_batch import AnthropicBatch
+        return AnthropicBatch(thinking=thinking)
+    if name == "vllm":
+        from .vllm_offline import VLLMOffline
+        if not model:
+            raise ValueError("vllm needs a model name")
+        return VLLMOffline(model)
+    if name == "gemini":
+        from .gemini_batch import GeminiBatch
+        return GeminiBatch()
+    if name == "bedrock":
+        raise NotImplementedError(
+            "the Bedrock adapter is next on the list; see docs/ROADMAP.md. Contributions welcome."
+        )
+    raise ValueError(f"unknown provider: {name}")
+
+
+__all__ = ["BatchProvider", "FakeProvider", "get_provider"]

steadybatch/providers/anthropic_batch.py

@@ -0,0 +1,95 @@
+"""Anthropic Message Batches API.
+
+Flow: create a batch with up to 100,000 requests in one call, poll until
+processing_status is "ended", then stream results. Each result is
+succeeded, errored, canceled or expired.
+
+When a JSON Schema is given, it is sent as a structured output format
+(`output_config.format`), so the reply is constrained to the schema, the
+same as the OpenAI and Gemini adapters. Pass `native_schema=False` to
+describe the schema in the system prompt instead. `thinking` ("disabled" or
+"adaptive") is sent when given; left out, the model uses its default, and on
+Claude Haiku 5.5 that default is adaptive thinking, which counts against
+max_tokens. `temperature` is not sent:
+current Claude models reject non-default sampling settings, and anthropic
+SDK 1.x no longer takes the parameter.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Iterator
+
+from ..models import BatchState, BatchStatus, PreparedRequest, RawResult
+from .base import BatchProvider
+
+
+class AnthropicBatch(BatchProvider):
+    name = "anthropic"
+    max_requests_per_batch = 100_000
+    max_bytes_per_batch = 256 * 1024 * 1024
+
+    def __init__(self, client=None, *, native_schema: bool = True, thinking: str | None = None):
+        if client is None:
+            import anthropic  # imported here so the package works without it
+            client = anthropic.Anthropic()
+        self.client = client
+        self.native_schema = native_schema
+        self.thinking = thinking
+
+    def to_line(self, req: PreparedRequest) -> dict:
+        params = {
+            "model": req.model,
+            "max_tokens": req.request.max_tokens,
+            "messages": req.request.messages,
+        }
+        if self.thinking:
+            params["thinking"] = {"type": self.thinking}
+        system = req.request.system or ""
+        if req.response_schema is not None and self.native_schema:
+            params["output_config"] = {"format": {"type": "json_schema", "schema": req.response_schema}}
+        elif req.response_schema is not None:
+            # Ask for JSON in the prompt; the runner checks every answer against the schema.
+            system = (system + "\n\nReply with only a JSON object that matches this JSON Schema:\n"
+                      + json.dumps(req.response_schema)).strip()
+        if system:
+            params["system"] = system
+        return {"custom_id": req.custom_id, "params": params}
+
+    def submit(self, batch: list[PreparedRequest]) -> str:
+        created = self.client.messages.batches.create(requests=[self.to_line(r) for r in batch])
+        return created.id
+
+    def is_over_capacity(self, exc: Exception) -> bool:
+        return getattr(exc, "status_code", None) == 429
+
+    def status(self, batch_id: str) -> BatchStatus:
+        b = self.client.messages.batches.retrieve(batch_id)
+        c = b.request_counts
+        if b.processing_status == "ended":
+            state = BatchState.DONE
+        elif b.processing_status == "canceling":
+            state = BatchState.FAILED
+        else:
+            state = BatchState.RUNNING
+        total = c.processing + c.succeeded + c.errored + c.canceled + c.expired
+        return BatchStatus(state, total=total, succeeded=c.succeeded,
+                           failed=c.errored + c.canceled + c.expired, detail=b.processing_status)
+
+    def results(self, batch_id: str) -> Iterator[RawResult]:
+        for entry in self.client.messages.batches.results(batch_id):
+            result = entry.result
+            if result.type == "succeeded":
+                msg = result.message
+                if getattr(msg, "stop_reason", None) == "refusal":
+                    yield RawResult(entry.custom_id, ok=False, error="refusal")
+                    continue
+                text = "".join(block.text for block in msg.content if block.type == "text")
+                truncated = getattr(msg, "stop_reason", None) == "max_tokens"
+                yield RawResult(entry.custom_id, ok=True, text=text,
+                                error="truncated (max_tokens)" if truncated else None,
+                                input_tokens=msg.usage.input_tokens, output_tokens=msg.usage.output_tokens)
+            elif result.type == "errored":
+                yield RawResult(entry.custom_id, ok=False, error=str(getattr(result, "error", "errored")))
+            else:
+                yield RawResult(entry.custom_id, ok=False, error=result.type)

steadybatch/providers/base.py

@@ -0,0 +1,60 @@
+"""The interface every batch provider implements.
+
+A provider only has to do four things: say what its limits are, submit a
+list of requests, report a batch's status, and hand back whatever lines it
+has. Everything else (retries, checks, ordering, checkpoints) lives in the
+runner, so every provider gets the same treatment and the comparison is fair.
+"""
+
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from typing import Iterator
+
+from ..models import BatchStatus, PreparedRequest, RawResult
+
+
+class BatchProvider(ABC):
+    name: str = "base"
+    max_requests_per_batch: int = 10_000
+    max_bytes_per_batch: int = 100 * 1024 * 1024
+
+    def size_of(self, req: PreparedRequest) -> int:
+        """Rough size of one request line in the upload file."""
+        return len(json.dumps(self.to_line(req)).encode("utf-8")) + 1
+
+    def to_line(self, req: PreparedRequest) -> dict:
+        """The provider-specific JSON for one request. Used for sizing and upload."""
+        return {"custom_id": req.custom_id, "messages": req.request.messages}
+
+    def is_over_capacity(self, exc: Exception) -> bool:
+        """True if `submit` failed because the account's queue or quota is full.
+
+        The runner then sends smaller batches instead of stopping. Providers that
+        can tell this apart from other errors override it.
+        """
+        return False
+
+    def is_transient(self, exc: Exception) -> bool:
+        """True if `submit` failed for a reason worth retrying as-is: a dropped
+        connection, a timeout or a server error. The SDKs retry a couple of times
+        on their own; a long job needs more patience than that."""
+        names = {c.__name__ for c in type(exc).__mro__}
+        if names & {"APIConnectionError", "APITimeoutError", "ConnectionError", "TimeoutError",
+                    "ServerError", "InternalServerError", "ServiceUnavailableError"}:
+            return True
+        status = getattr(exc, "status_code", None) or getattr(exc, "code", None)
+        return isinstance(status, int) and status >= 500
+
+    @abstractmethod
+    def submit(self, batch: list[PreparedRequest]) -> str:
+        """Send a batch. Return the provider's batch ID."""
+
+    @abstractmethod
+    def status(self, batch_id: str) -> BatchStatus:
+        """Report where a batch is."""
+
+    @abstractmethod
+    def results(self, batch_id: str) -> Iterator[RawResult]:
+        """Yield every line the provider returned, successes and errors alike."""