hmft 0.2.0__py3-none-any.whl

hmft/__init__.py

@@ -0,0 +1,15 @@
+"""hmft: how much time will this LLM call take?"""
+__version__ = "0.2.0"
+
+from .estimate import Estimate, estimate  # noqa: E402
+from .length import Band, HeuristicLengthEstimator, LengthEstimator, LengthOptions  # noqa: E402
+from .stream import ETA, LiveETA  # noqa: E402
+from .throughput import ThroughputEntry, ThroughputTable, lookup  # noqa: E402
+from .wrap import Actual, CallResult, TrackedStream, wrap_anthropic, wrap_openai  # noqa: E402
+
+__all__ = [
+    "estimate", "Estimate", "Band", "ETA", "LiveETA",
+    "HeuristicLengthEstimator", "LengthEstimator", "LengthOptions",
+    "ThroughputEntry", "ThroughputTable", "lookup", "__version__",
+    "wrap_openai", "wrap_anthropic", "Actual", "CallResult", "TrackedStream",
+]

hmft/data/throughput.json

@@ -0,0 +1,272 @@
+{
+  "schema_version": 1,
+  "updated_at": "2026-09-16T17:58:00Z",
+  "entries": {
+    "openai/_default": {
+      "provider": "openai",
+      "model": "_default",
+      "measured": true,
+      "samples": 20,
+      "measured_at": "2026-09-16T07:26:07Z",
+      "ttft_s": {
+        "p50": 0.725,
+        "p90": 2.229
+      },
+      "tokens_per_s": {
+        "p50": 125.2,
+        "p10": 90.5
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.0,
+      "reasoning_tokens_p50": null,
+      "source": "scripts/benchmark.py (aggregate of measured models)",
+      "notes": "provider fallback: median p50 / worst slow side across gpt-4.1, gpt-4.1-mini"
+    },
+    "openai/gpt-4.1-mini": {
+      "provider": "openai",
+      "model": "gpt-4.1-mini",
+      "measured": true,
+      "samples": 10,
+      "measured_at": "2026-09-16T07:21:48Z",
+      "ttft_s": {
+        "p50": 0.69,
+        "p90": 1.622
+      },
+      "tokens_per_s": {
+        "p50": 139.2,
+        "p10": 119.1
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.0,
+      "reasoning_tokens_p50": null,
+      "source": "scripts/benchmark.py",
+      "notes": ""
+    },
+    "openai/gpt-4.1": {
+      "provider": "openai",
+      "model": "gpt-4.1",
+      "measured": true,
+      "samples": 10,
+      "measured_at": "2026-09-16T07:26:07Z",
+      "ttft_s": {
+        "p50": 0.76,
+        "p90": 2.229
+      },
+      "tokens_per_s": {
+        "p50": 111.3,
+        "p10": 90.5
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.0,
+      "reasoning_tokens_p50": null,
+      "source": "scripts/benchmark.py",
+      "notes": "slope from median method (raw samples of this run not re-processed)"
+    },
+    "openai/gpt-5-mini": {
+      "provider": "openai",
+      "model": "gpt-5-mini",
+      "measured": true,
+      "samples": 10,
+      "measured_at": "2026-09-16T07:21:48Z",
+      "ttft_s": {
+        "p50": 22.94,
+        "p90": 39.774
+      },
+      "tokens_per_s": {
+        "p50": 75.2,
+        "p10": 67.1
+      },
+      "ttft_per_1k_prompt_tokens_s": 1.9401,
+      "reasoning_tokens_p50": 1824,
+      "source": "scripts/benchmark.py",
+      "notes": "reasoning model: ttft includes hidden thinking time for this prompt shape; slope unreliable"
+    },
+    "openai/gpt-5@low": {
+      "provider": "openai",
+      "model": "gpt-5@low",
+      "measured": true,
+      "samples": 10,
+      "measured_at": "2026-09-16T07:21:48Z",
+      "ttft_s": {
+        "p50": 9.5,
+        "p90": 17.52
+      },
+      "tokens_per_s": {
+        "p50": 123.0,
+        "p10": 69.4
+      },
+      "ttft_per_1k_prompt_tokens_s": 5.9433,
+      "reasoning_tokens_p50": 1312,
+      "source": "scripts/benchmark.py",
+      "notes": "reasoning model at reasoning_effort=low; reasoning model: ttft includes hidden thinking time for this prompt shape; slope unreliable"
+    },
+    "openai/gpt-5@medium": {
+      "provider": "openai",
+      "model": "gpt-5@medium",
+      "measured": true,
+      "samples": 10,
+      "measured_at": "2026-09-16T07:21:48Z",
+      "ttft_s": {
+        "p50": 24.51,
+        "p90": 26.682
+      },
+      "tokens_per_s": {
+        "p50": 169.1,
+        "p10": 149.6
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.0,
+      "reasoning_tokens_p50": 1760,
+      "source": "scripts/benchmark.py",
+      "notes": "reasoning model at reasoning_effort=medium; reasoning model: ttft includes hidden thinking time for this prompt shape; slope unreliable"
+    },
+    "anthropic/_default": {
+      "provider": "anthropic",
+      "model": "_default",
+      "measured": true,
+      "samples": 50,
+      "measured_at": "2026-09-16T17:58:00Z",
+      "ttft_s": {
+        "p50": 0.765,
+        "p90": 1.355
+      },
+      "tokens_per_s": {
+        "p50": 60.1,
+        "p10": 34.6
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.2326,
+      "reasoning_tokens_p50": null,
+      "source": "scripts/benchmark.py (aggregate of measured models)",
+      "notes": "provider fallback: median p50 / worst slow side across claude-haiku-4-5, claude-sonnet-4-5"
+    },
+    "anthropic/claude-sonnet-4-5": {
+      "provider": "anthropic",
+      "model": "claude-sonnet-4-5",
+      "measured": true,
+      "samples": 40,
+      "measured_at": "2026-09-16T17:58:00Z",
+      "ttft_s": {
+        "p50": 0.86,
+        "p90": 1.355
+      },
+      "tokens_per_s": {
+        "p50": 36.8,
+        "p10": 34.6
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.2326,
+      "reasoning_tokens_p50": null,
+      "source": "scripts/benchmark.py",
+      "notes": "two 10-run benchmarks pooled (17:49Z and 17:58Z)"
+    },
+    "anthropic/claude-haiku-4-5": {
+      "provider": "anthropic",
+      "model": "claude-haiku-4-5",
+      "measured": true,
+      "samples": 10,
+      "measured_at": "2026-09-16T07:21:48Z",
+      "ttft_s": {
+        "p50": 0.67,
+        "p90": 0.686
+      },
+      "tokens_per_s": {
+        "p50": 83.4,
+        "p10": 79.5
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.0,
+      "reasoning_tokens_p50": null,
+      "source": "scripts/benchmark.py",
+      "notes": ""
+    },
+    "openrouter/openai/gpt-4.1-mini": {
+      "provider": "openrouter",
+      "model": "openai/gpt-4.1-mini",
+      "measured": true,
+      "samples": 10,
+      "measured_at": "2026-09-16T07:21:48Z",
+      "ttft_s": {
+        "p50": 0.81,
+        "p90": 1.17
+      },
+      "tokens_per_s": {
+        "p50": 108.1,
+        "p10": 82.7
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.171,
+      "reasoning_tokens_p50": null,
+      "source": "scripts/benchmark.py",
+      "notes": ""
+    },
+    "openrouter/openai/gpt-4.1": {
+      "provider": "openrouter",
+      "model": "openai/gpt-4.1",
+      "measured": true,
+      "samples": 10,
+      "measured_at": "2026-09-16T07:21:48Z",
+      "ttft_s": {
+        "p50": 0.85,
+        "p90": 1.024
+      },
+      "tokens_per_s": {
+        "p50": 138.6,
+        "p10": 87.2
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.1603,
+      "reasoning_tokens_p50": null,
+      "source": "scripts/benchmark.py",
+      "notes": ""
+    },
+    "openrouter/openai/gpt-5-mini": {
+      "provider": "openrouter",
+      "model": "openai/gpt-5-mini",
+      "measured": true,
+      "samples": 10,
+      "measured_at": "2026-09-16T07:33:42Z",
+      "ttft_s": {
+        "p50": 32.84,
+        "p90": 41.578
+      },
+      "tokens_per_s": {
+        "p50": 84.9,
+        "p10": 55.9
+      },
+      "ttft_per_1k_prompt_tokens_s": 2.3944,
+      "reasoning_tokens_p50": 1536,
+      "source": "scripts/benchmark.py",
+      "notes": "reasoning model: ttft includes hidden thinking time for this prompt shape; slope unreliable"
+    },
+    "openrouter/anthropic/claude-sonnet-4.5": {
+      "provider": "openrouter",
+      "model": "anthropic/claude-sonnet-4.5",
+      "measured": true,
+      "samples": 6,
+      "measured_at": "2026-09-16T07:21:48Z",
+      "ttft_s": {
+        "p50": 1.96,
+        "p90": 3.024
+      },
+      "tokens_per_s": {
+        "p50": 39.8,
+        "p10": 34.2
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.3149,
+      "reasoning_tokens_p50": null,
+      "source": "scripts/benchmark.py",
+      "notes": ""
+    },
+    "openrouter/anthropic/claude-haiku-4.5": {
+      "provider": "openrouter",
+      "model": "anthropic/claude-haiku-4.5",
+      "measured": true,
+      "samples": 10,
+      "measured_at": "2026-09-16T07:33:42Z",
+      "ttft_s": {
+        "p50": 1.13,
+        "p90": 1.334
+      },
+      "tokens_per_s": {
+        "p50": 88.7,
+        "p10": 85.2
+      },
+      "ttft_per_1k_prompt_tokens_s": 0.0339,
+      "reasoning_tokens_p50": null,
+      "source": "scripts/benchmark.py",
+      "notes": ""
+    }
+  }
+}

hmft/estimate.py

@@ -0,0 +1,113 @@
+"""Pre-call estimate: how long will this call take?
+
+    total ≈ TTFT(prompt_tokens, model, provider) + output_tokens / tokens_per_s
+
+Every number is a Band(p50, p90) in the *time* sense: p90 is the slow side.
+The throughput table stores tokens_per_s as p50/p10 because for throughput
+the slow side is the LOW percentile. So:
+
+    time p50 = ttft.p50 + output.p50 / tps.p50
+    time p90 = ttft.p90 + output.p90 / tps.p10     <- throughput p10, not p90
+
+If the table has no usable entry for the model or its provider default, the
+token estimate is still returned and the time fields are None with
+basis="unmeasured". Nothing is invented.
+"""
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any, Iterable, Optional
+
+from .length import DEFAULT_LENGTH_ESTIMATOR, Band, LengthEstimator, LengthOptions
+from .throughput import ThroughputEntry, lookup
+from .tokens import count_messages
+
+# Reasoning models emit hidden thinking tokens that are billed and take wall
+# time but are not in the visible output. v1 does not model them; the
+# benchmark measures the damage per effort level (see throughput.json notes).
+REASONING_EFFORT_SUFFIX = "@"
+# Bare reasoning-model calls run at the provider's default effort. Rows are keyed per effort, so a call
+# with no effort set falls back to the @medium row when no bare row exists (gpt-5 -> gpt-5@medium).
+DEFAULT_REASONING_EFFORT = "medium"
+
+
+@dataclass(frozen=True)
+class Estimate:
+    ttft_s: Optional[Band]
+    output_tokens: Band
+    total_s: Optional[Band]
+    prompt_tokens: int
+    model: str
+    provider: str
+    basis: str          # "measured" | "provider_default" | "unmeasured"
+    basis_key: str      # throughput table key that was used, "" if none
+    reasoning_effort: Optional[str] = None  # effort requested, or the one the table row implies (gpt-5 -> medium); None for non-reasoning
+    effort_source: Optional[str] = None     # "requested" | "default" when reasoning_effort is set, else None
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def __str__(self) -> str:
+        if self.total_s is None:
+            return (f"~{self.output_tokens.p50:.0f} output tokens (p90 {self.output_tokens.p90:.0f}); "
+                    f"no timing for {self.provider}/{self.model} (unmeasured)")
+        return (f"~{self.total_s.p50:.1f}s (p90 {self.total_s.p90:.1f}s): "
+                f"ttft {self.ttft_s.p50:.2f}s, ~{self.output_tokens.p50:.0f} tokens [{self.basis}]")
+
+
+def table_key_for(model: str, reasoning_effort: Optional[str]) -> str:
+    """gpt-5 + effort 'low' -> 'gpt-5@low'. Table entries for reasoning models are per effort."""
+    return f"{model}{REASONING_EFFORT_SUFFIX}{reasoning_effort}" if reasoning_effort else model
+
+
+def table_keys_for(model: str, reasoning_effort: Optional[str]) -> list[str]:
+    """Lookup order. With an effort: just that row. Without: the bare row, then @medium."""
+    if reasoning_effort:
+        return [table_key_for(model, reasoning_effort)]
+    return [model, table_key_for(model, DEFAULT_REASONING_EFFORT)]
+
+
+def _ttft_band(entry: ThroughputEntry, prompt_tokens: int) -> Band:
+    slope = entry.ttft_per_1k_prompt_tokens_s or 0.0
+    extra = slope * prompt_tokens / 1000.0
+    return Band(entry.ttft_p50_s + extra, entry.ttft_p90_s + extra * 1.5)
+
+
+def estimate(
+    messages: Iterable[Any],
+    model: str,
+    provider: str = "openai",
+    *,
+    system: Optional[str] = None,
+    max_tokens: Optional[int] = None,
+    json_mode: bool = False,
+    tool_choice_forced: bool = False,
+    reasoning_effort: Optional[str] = None,
+    length_estimator: Optional[LengthEstimator] = None,
+    throughput_entry: Optional[ThroughputEntry] = None,
+) -> Estimate:
+    """Estimate time and output size for a chat call before sending it.
+
+    `throughput_entry` lets callers (and tests) bypass the table lookup.
+    `length_estimator` is the v2 plug point.
+    """
+    msgs = list(messages)
+    prompt_tokens = count_messages(msgs, model, provider, system=system)
+    options = LengthOptions(max_tokens=max_tokens, json_mode=json_mode, tool_choice_forced=tool_choice_forced, system=system)
+    output = (length_estimator or DEFAULT_LENGTH_ESTIMATOR).estimate(msgs, model, options)
+
+    entry = throughput_entry if throughput_entry is not None else lookup(model, provider, table_keys_for(model, reasoning_effort))
+    effort, source = reasoning_effort, ("requested" if reasoning_effort else None)
+    if effort is None and REASONING_EFFORT_SUFFIX in entry.basis_key:
+        # the caller set nothing; the provider's default effort applies, and the table row says which one we assumed
+        effort, source = entry.basis_key.rsplit(REASONING_EFFORT_SUFFIX, 1)[1], "default"
+    if not entry.usable:
+        return Estimate(None, output, None, prompt_tokens, model, provider, "unmeasured", entry.basis_key, effort, source)
+
+    ttft = _ttft_band(entry, prompt_tokens)
+    total = Band(
+        ttft.p50 + output.p50 / entry.tps_p50,
+        ttft.p90 + output.p90 / entry.tps_p10,  # slow throughput -> long time
+    )
+    basis = "provider_default" if entry.model == "_default" else "measured"
+    return Estimate(ttft, output, total, prompt_tokens, model, provider, basis, entry.basis_key, effort, source)