costplan 0.1.0__py3-none-any.whl

costplan/__init__.py

@@ -0,0 +1,36 @@
+"""How much will my agent cost? Plan agent budgets from a handful of pilot runs."""
+
+from .loading import LoadedRuns, load_runs
+from .planning import (
+    CapAdvice,
+    CostComparison,
+    CostSummary,
+    SpendProjection,
+    compare_cost_per_success,
+    guaranteed_stop_rate,
+    pilot_runs_for_average,
+    project_spend,
+    recommend_cap,
+    run_cost,
+    runs_needed_for_cap,
+    summarize,
+)
+
+__all__ = [
+    "load_runs",
+    "LoadedRuns",
+    "run_cost",
+    "summarize",
+    "recommend_cap",
+    "runs_needed_for_cap",
+    "guaranteed_stop_rate",
+    "pilot_runs_for_average",
+    "project_spend",
+    "compare_cost_per_success",
+    "CostSummary",
+    "CapAdvice",
+    "SpendProjection",
+    "CostComparison",
+]
+
+__version__ = "0.1.0"

costplan/_stats.py

@@ -0,0 +1,109 @@
+"""Small statistics helpers built on the standard library only."""
+
+from __future__ import annotations
+
+import math
+import random
+from statistics import NormalDist
+from typing import Callable, List, Optional, Sequence, Tuple
+
+_NORMAL = NormalDist()
+
+
+def check_level(value: float, name: str) -> None:
+    if not 0 < value < 1:
+        raise ValueError(f"{name} must be strictly between 0 and 1, got {value!r}")
+
+
+def z_two_sided(confidence: float) -> float:
+    return _NORMAL.inv_cdf(1 - (1 - confidence) / 2)
+
+
+def wilson(successes: int, n: int, confidence: float) -> Tuple[float, float]:
+    z = z_two_sided(confidence)
+    p = successes / n
+    denom = 1 + z * z / n
+    centre = (p + z * z / (2 * n)) / denom
+    half = z / denom * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
+    return max(0.0, centre - half), min(1.0, centre + half)
+
+
+def binom_cdf_table(n: int, p: float) -> List[float]:
+    """[P(X <= k) for k = 0..n] for X ~ Binomial(n, p), computed in log space."""
+    if p <= 0:
+        return [1.0] * (n + 1)
+    if p >= 1:
+        return [0.0] * n + [1.0]
+    log_p, log_q = math.log(p), math.log1p(-p)
+    log_n = math.lgamma(n + 1)
+    table, total = [], 0.0
+    for i in range(n + 1):
+        total += math.exp(log_n - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * log_p + (n - i) * log_q)
+        table.append(min(1.0, total))
+    return table
+
+
+def quantile(sorted_values: Sequence[float], p: float) -> float:
+    """Linear interpolation quantile (the usual default in numpy and pandas)."""
+    n = len(sorted_values)
+    if n == 1:
+        return sorted_values[0]
+    pos = p * (n - 1)
+    lo = math.floor(pos)
+    hi = min(lo + 1, n - 1)
+    return sorted_values[lo] + (sorted_values[hi] - sorted_values[lo]) * (pos - lo)
+
+
+def quantile_interval(
+    sorted_values: Sequence[float], p: float, confidence: float
+) -> Tuple[Optional[float], Optional[float]]:
+    """Distribution-free interval for the p-th quantile from order statistics.
+
+    Makes no assumption about the shape of the cost distribution. Either end
+    is None when there are too few runs to bound it at this confidence.
+    """
+    n = len(sorted_values)
+    cdf = binom_cdf_table(n, p)
+    tail = (1 - confidence) / 2
+    low = None
+    for k in range(n, 0, -1):  # largest k with P(X_(k) > q) at most the tail
+        if cdf[k - 1] <= tail:
+            low = sorted_values[k - 1]
+            break
+    high = upper_quantile_bound(sorted_values, p, 1 - tail, cdf)
+    return low, high
+
+
+def upper_quantile_bound(
+    sorted_values: Sequence[float], p: float, confidence: float, cdf: Optional[List[float]] = None
+) -> Optional[float]:
+    """Smallest order statistic that is above the p-th quantile with the given confidence."""
+    n = len(sorted_values)
+    cdf = cdf or binom_cdf_table(n, p)
+    for k in range(1, n + 1):
+        if cdf[k - 1] >= confidence:
+            return sorted_values[k - 1]
+    return None
+
+
+def bootstrap_interval(
+    columns: Sequence[Sequence],
+    statistic: Callable[..., Optional[float]],
+    confidence: float,
+    resamples: int,
+    seed: int,
+) -> Tuple[float, float]:
+    """Percentile bootstrap, resampling rows of the given columns together."""
+    rng = random.Random(seed)
+    n = len(columns[0])
+    values: List[float] = []
+    for _ in range(resamples):
+        idx = [rng.randrange(n) for _ in range(n)]
+        value = statistic(*[[col[i] for i in idx] for col in columns])
+        if value is not None and math.isfinite(value):
+            values.append(value)
+    if not values:
+        return float("nan"), float("nan")
+    values.sort()
+    tail = (1 - confidence) / 2
+    return quantile(values, tail), quantile(values, 1 - tail)

costplan/cli.py

@@ -0,0 +1,165 @@
+"""Command line: ``costplan summary``, ``costplan cap``, ``costplan project`` and more."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import sys
+from typing import List, Optional, Tuple
+
+from . import (
+    LoadedRuns,
+    compare_cost_per_success,
+    load_runs,
+    guaranteed_stop_rate,
+    pilot_runs_for_average,
+    project_spend,
+    recommend_cap,
+    runs_needed_for_cap,
+    summarize,
+)
+
+def read_runs(path: str, args: argparse.Namespace) -> Tuple[List[float], Optional[List[bool]]]:
+    loaded = _load(path, args)
+    return loaded.costs, loaded.successes
+
+
+def _load(path: str, args: argparse.Namespace, group_by: Optional[str] = None) -> LoadedRuns:
+    loaded = load_runs(
+        path,
+        cost=args.cost_column,
+        input_tokens=args.input_column,
+        output_tokens=args.output_column,
+        success=args.success_column,
+        group_by=group_by if group_by is not None else args.group_by,
+        input_price=args.input_price,
+        output_price=args.output_price,
+        cached_price=args.cached_price,
+    )
+    if args.verbose:
+        print(loaded.report(), file=sys.stderr)
+    return loaded
+
+
+def _cmd_summary(args: argparse.Namespace) -> str:
+    costs, flags = read_runs(args.file, args)
+    return summarize(costs, flags, args.confidence).summary()
+
+
+def _cmd_cap(args: argparse.Namespace) -> str:
+    costs, _ = read_runs(args.file, args)
+    return recommend_cap(costs, args.stop_rate, args.confidence).summary()
+
+
+def _cmd_project(args: argparse.Namespace) -> str:
+    costs, _ = read_runs(args.file, args)
+    return project_spend(costs, args.runs, args.confidence).summary()
+
+
+def _cmd_compare(args: argparse.Namespace) -> str:
+    key = args.task_column
+    if args.paired and key and not args.group_by:
+        # Group by the task column so the two files can be lined up task by task.
+        a, b = _load(args.file_a, args, key), _load(args.file_b, args, key)
+    else:
+        a, b = _load(args.file_a, args), _load(args.file_b, args)
+    if a.successes is None or b.successes is None:
+        raise ValueError("both files need a success, status or error column to compare cost per success")
+    a_costs, a_flags, b_costs, b_flags = a.costs, a.successes, b.costs, b.successes
+    if args.paired and a.run_ids is not None and b.run_ids is not None:
+        b_index = {run: i for i, run in enumerate(b.run_ids)}
+        shared = [i for i, run in enumerate(a.run_ids) if run in b_index]
+        if len(shared) < 2:
+            raise ValueError("the two files share fewer than two task ids")
+        a_costs = [a.costs[i] for i in shared]
+        a_flags = [a.successes[i] for i in shared]
+        b_costs = [b.costs[b_index[a.run_ids[i]]] for i in shared]
+        b_flags = [b.successes[b_index[a.run_ids[i]]] for i in shared]
+    names = tuple(args.names) if args.names else (args.file_a, args.file_b)
+    return compare_cost_per_success(
+        a_costs, a_flags, b_costs, b_flags, names, paired=args.paired, confidence=args.confidence
+    ).summary()
+
+
+def _cmd_pilot(args: argparse.Namespace) -> str:
+    lines = []
+    n = runs_needed_for_cap(args.stop_rate, args.confidence)
+    lines.append(
+        f"To set a cap that stops at most {args.stop_rate:.1%} of normal runs ({args.confidence:.0%} confidence): "
+        f"{n} pilot runs."
+    )
+    for size in (20, 50, 100):
+        if size < n:
+            lines.append(f"  With {size} runs, the best promise is {guaranteed_stop_rate(size, args.confidence):.1%}.")
+    if args.spread is not None:
+        m = pilot_runs_for_average(args.spread, args.margin, args.confidence)
+        lines.append(f"To pin the average cost to within {args.margin:.0%} (spread {args.spread:g}): {m} pilot runs.")
+    return "\n".join(lines)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="costplan",
+        description="How much will my agent cost? Plan budgets, caps and spend from pilot runs.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    def file_options(p: argparse.ArgumentParser) -> None:
+        p.add_argument("--cost-column", help="column with each row's dollar cost (found automatically if left out)")
+        p.add_argument("--input-column", help="column with input tokens (found automatically if left out)")
+        p.add_argument("--output-column", help="column with output tokens (found automatically if left out)")
+        p.add_argument("--success-column", help="column with pass/fail (found automatically if left out)")
+        p.add_argument("--group-by", help="column naming the run each row belongs to, for exports with one row per model call")
+        p.add_argument("-v", "--verbose", action="store_true", help="say which columns were used")
+        p.add_argument("--input-price", type=float, help="dollars per million input tokens, if the file has tokens not costs")
+        p.add_argument("--output-price", type=float, help="dollars per million output tokens")
+        p.add_argument("--cached-price", type=float, help="dollars per million cached input tokens")
+        p.add_argument("--confidence", type=float, default=0.95)
+
+    p = sub.add_parser("summary", help="average, typical and expensive runs, success rate, cost per success")
+    p.add_argument("file", help="CSV with one row per pilot run")
+    file_options(p)
+    p.set_defaults(run=_cmd_summary)
+
+    p = sub.add_parser("cap", help="what per-run budget cap to set")
+    p.add_argument("file")
+    p.add_argument("--stop-rate", type=float, default=0.02, help="largest share of normal runs the cap may stop")
+    file_options(p)
+    p.set_defaults(run=_cmd_cap)
+
+    p = sub.add_parser("project", help="total spend for a number of future runs")
+    p.add_argument("file")
+    p.add_argument("--runs", type=int, required=True, help="number of future runs, e.g. a month of traffic")
+    file_options(p)
+    p.set_defaults(run=_cmd_project)
+
+    p = sub.add_parser("compare", help="which setup is cheaper per successful task")
+    p.add_argument("file_a")
+    p.add_argument("file_b")
+    p.add_argument("--names", nargs=2, metavar=("A", "B"))
+    p.add_argument("--paired", action="store_true", help="both files ran the same tasks; compare task by task")
+    p.add_argument("--task-column", help="column with a task id, to line up the two files when --paired")
+    file_options(p)
+    p.set_defaults(run=_cmd_compare)
+
+    p = sub.add_parser("pilot", help="how many pilot runs to collect")
+    p.add_argument("--stop-rate", type=float, default=0.02)
+    p.add_argument("--spread", type=float, help="cost spread (standard deviation / average), from a small pilot")
+    p.add_argument("--margin", type=float, default=0.10, help="relative precision wanted for the average")
+    p.add_argument("--confidence", type=float, default=0.95)
+    p.set_defaults(run=_cmd_pilot)
+    return parser
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        print(args.run(args))
+    except (ValueError, OSError) as exc:
+        print(f"costplan: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

costplan/loading.py

@@ -0,0 +1,264 @@
+"""Read run costs straight from tracing and usage exports.
+
+Exports come in two shapes:
+
+* **One row per run** (a trace, a session, a task), with the run's total
+  cost already added up. Load it as it is.
+* **One row per model call**, with a column naming the run each call belongs
+  to. Pass ``group_by`` with that column, and the calls are added up into runs.
+
+Do not group an export that already has run totals, or the same cost is
+counted twice. Some tools export both a run and its calls in one file; filter
+it to one or the other first.
+
+Column names are found automatically from common spellings (for example
+``cost``, ``total_cost``, ``totalCost``, ``calculatedTotalCost``,
+``prompt_tokens``, ``input_tokens``, ``usage.input``, ``trace_id``,
+``traceId``). Pass the column names yourself whenever the guess is wrong.
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+from .planning import run_cost
+
+COST_NAMES = ["cost", "totalcost", "calculatedtotalcost", "costusd", "usd", "totalcostusd"]
+INPUT_NAMES = ["inputtokens", "prompttokens", "usageinput", "usageinputtokens", "usageprompttokens"]
+OUTPUT_NAMES = ["outputtokens", "completiontokens", "usageoutput", "usageoutputtokens", "usagecompletiontokens"]
+CACHED_NAMES = ["cachedtokens", "cachereadinputtokens", "cachereadtokens"]
+SUCCESS_NAMES = ["success", "succeeded", "passed", "pass", "issuccess", "ok"]
+STATUS_NAMES = ["status"]
+ERROR_NAMES = ["error"]
+
+_TRUE = {"1", "true", "t", "yes", "y", "pass", "passed", "success", "succeeded", "ok", "completed", "done"}
+_FALSE = {"0", "false", "f", "no", "n", "fail", "failed", "failure", "error", "errored", "cancelled", "timeout"}
+
+
+def _norm(name: str) -> str:
+    return re.sub(r"[^a-z0-9]", "", name.lower())
+
+
+def _flatten(record: Dict[str, Any], prefix: str = "") -> Dict[str, Any]:
+    out: Dict[str, Any] = {}
+    for key, value in record.items():
+        full = f"{prefix}.{key}" if prefix else str(key)
+        if isinstance(value, dict):
+            out.update(_flatten(value, full))
+        else:
+            out[full] = value
+    return out
+
+
+def _read_records(path: Path) -> List[Dict[str, Any]]:
+    text = path.read_text(encoding="utf-8-sig")
+    suffix = path.suffix.lower()
+    if suffix == ".csv":
+        return [dict(row) for row in csv.DictReader(text.splitlines())]
+    if suffix in {".jsonl", ".ndjson"}:
+        records = [json.loads(line) for line in text.splitlines() if line.strip()]
+    elif suffix == ".json":
+        data = json.loads(text)
+        if isinstance(data, dict):
+            for key in ("data", "runs", "traces", "observations", "items", "results"):
+                if isinstance(data.get(key), list):
+                    data = data[key]
+                    break
+        if not isinstance(data, list):
+            raise ValueError(f"{path}: expected a list of records, or an object with a 'data' list")
+        records = data
+    else:
+        raise ValueError(f"{path}: use a .csv, .json or .jsonl file")
+    if not all(isinstance(r, dict) for r in records):
+        raise ValueError(f"{path}: every record must be an object")
+    return [_flatten(r) for r in records]
+
+
+def _find(columns: Iterable[str], names: List[str]) -> Optional[str]:
+    """First column whose full name, or last dotted part, matches one of the spellings."""
+    columns = list(columns)
+    for wanted in names:
+        for col in columns:
+            if _norm(col) == wanted:
+                return col
+    for wanted in names:
+        for col in columns:
+            if "." in col and _norm(col.rsplit(".", 1)[-1]) == wanted:
+                return col
+    return None
+
+
+def _number(value: Any, what: str, where: str) -> Optional[float]:
+    if value is None:
+        return None
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return float(value)
+    text = str(value).strip().lstrip("$").replace(",", "")
+    if text == "" or text.lower() in {"none", "null", "nan"}:
+        return None
+    try:
+        return float(text)
+    except ValueError:
+        raise ValueError(f"{where}: cannot read {what} {value!r} as a number") from None
+
+
+def _outcome(row: Dict[str, Any], success_col, status_col, error_col, where: str) -> Optional[bool]:
+    if success_col is not None:
+        cell = str(row.get(success_col, "")).strip().lower()
+        if cell in _TRUE:
+            return True
+        if cell in _FALSE:
+            return False
+        if cell in {"", "none", "null"}:
+            return None
+        raise ValueError(f"{where}: cannot read {row.get(success_col)!r} as success or failure")
+    if status_col is not None:
+        cell = str(row.get(status_col, "")).strip().lower()
+        if cell in _TRUE:
+            return True
+        if cell in _FALSE:
+            return False
+        return None
+    if error_col is not None:
+        value = row.get(error_col)
+        return value is None or str(value).strip().lower() in {"", "none", "null", "false"}
+    return None
+
+
+@dataclass
+class LoadedRuns:
+    """Run costs and outcomes read from an export, plus what was used to read them."""
+
+    costs: List[float]
+    successes: Optional[List[bool]]
+    run_ids: Optional[List[str]]
+    rows: int
+    columns_used: Dict[str, str] = field(default_factory=dict)
+    rows_without_cost: int = 0
+
+    def report(self) -> str:
+        used = ", ".join(f"{k} from '{v}'" for k, v in self.columns_used.items())
+        lines = [f"Read {self.rows} rows into {len(self.costs)} runs ({used})."]
+        if self.rows_without_cost:
+            lines.append(
+                f"{self.rows_without_cost} rows had no cost and count as $0. If your tool could not price "
+                "those calls, pass token columns and prices instead."
+            )
+        if self.successes is None:
+            lines.append("No success column found, so cost per successful task is not available.")
+        return "\n".join(lines)
+
+
+def load_runs(
+    path,
+    *,
+    cost: Optional[str] = None,
+    input_tokens: Optional[str] = None,
+    output_tokens: Optional[str] = None,
+    cached_tokens: Optional[str] = None,
+    success: Optional[str] = None,
+    group_by: Optional[str] = None,
+    input_price: Optional[float] = None,
+    output_price: Optional[float] = None,
+    cached_price: Optional[float] = None,
+) -> LoadedRuns:
+    """Read run costs from a CSV, JSON or JSON Lines export.
+
+    Args:
+        path: the export file.
+        cost, input_tokens, output_tokens, cached_tokens, success: column
+            names, if the automatic guess is wrong. Nested JSON fields are
+            named with dots, for example ``usage.prompt_tokens``.
+        group_by: column naming the run each row belongs to, for exports with
+            one row per model call. Costs are added up per run, and a run
+            fails if any of its rows failed.
+        input_price, output_price, cached_price: dollars per million tokens,
+            used when there is no cost column, or for rows with no cost.
+
+    Without a success column, ``status`` (success or error) or ``error``
+    (empty means success) are used if present.
+    """
+    path = Path(path)
+    records = _read_records(path)
+    if not records:
+        raise ValueError(f"{path} has no rows")
+    columns = list({k: None for r in records for k in r}.keys())
+
+    def pick(explicit, names, label):
+        if explicit is None:
+            return _find(columns, names)
+        if explicit not in columns:
+            raise ValueError(f"{path} has no {label} column named {explicit!r}")
+        return explicit
+
+    cost_col = pick(cost, COST_NAMES, "cost")
+    in_col = pick(input_tokens, INPUT_NAMES, "input token")
+    out_col = pick(output_tokens, OUTPUT_NAMES, "output token")
+    cached_col = pick(cached_tokens, CACHED_NAMES, "cached token")
+    success_col = pick(success, SUCCESS_NAMES, "success")
+    status_col = _find(columns, STATUS_NAMES) if success_col is None else None
+    error_col = _find(columns, ERROR_NAMES) if success_col is None and status_col is None else None
+    if group_by is not None and group_by not in columns:
+        raise ValueError(f"{path} has no column named {group_by!r} to group by")
+    have_prices = input_price is not None and output_price is not None
+    if cost_col is None and not (in_col and out_col and have_prices):
+        if in_col and out_col:
+            raise ValueError(
+                f"{path} has token columns but no cost column: give prices in dollars per million tokens "
+                "(--input-price and --output-price, or input_price and output_price in Python)"
+            )
+        raise ValueError(
+            f"{path}: found no cost column and no token columns. Columns are: {', '.join(columns[:15])}"
+        )
+
+    per_row: List[Tuple[Optional[str], float, Optional[bool]]] = []
+    missing = priced_by_tokens = 0
+    for i, row in enumerate(records, start=1):
+        where = f"{path}, row {i}"
+        value = _number(row.get(cost_col), "cost", where) if cost_col else None
+        if value is None and in_col and out_col and have_prices:
+            tokens_in = _number(row.get(in_col), "input tokens", where) or 0.0
+            tokens_out = _number(row.get(out_col), "output tokens", where) or 0.0
+            tokens_cached = (_number(row.get(cached_col), "cached tokens", where) or 0.0) if cached_col else 0.0
+            value = run_cost(tokens_in, tokens_out, input_price, output_price, min(tokens_cached, tokens_in), cached_price)
+            priced_by_tokens += 1
+        if value is None:
+            missing += 1
+            value = 0.0
+        if value < 0:
+            raise ValueError(f"{where}: cost must not be negative")
+        run = str(row.get(group_by)) if group_by else None
+        per_row.append((run, value, _outcome(row, success_col, status_col, error_col, where)))
+
+    if group_by:
+        totals: Dict[str, float] = {}
+        outcomes: Dict[str, Optional[bool]] = {}
+        for run, value, ok in per_row:
+            totals[run] = totals.get(run, 0.0) + value
+            if ok is not None:
+                outcomes[run] = (outcomes.get(run, True) and ok)
+        run_ids = list(totals)
+        costs = [totals[r] for r in run_ids]
+        flags = [outcomes[r] for r in run_ids] if len(outcomes) == len(run_ids) else None
+    else:
+        run_ids = None
+        costs = [value for _, value, _ in per_row]
+        flags_raw = [ok for _, _, ok in per_row]
+        flags = flags_raw if all(ok is not None for ok in flags_raw) else None
+
+    used: Dict[str, str] = {}
+    if cost_col:
+        used["cost"] = cost_col
+    if priced_by_tokens:
+        used["tokens"] = f"{in_col} and {out_col}"
+    for label, col in (("success", success_col), ("status", status_col), ("error", error_col)):
+        if col and flags is not None:
+            used[label] = col
+    if group_by:
+        used["runs"] = group_by
+    return LoadedRuns(costs, flags, run_ids, len(records), used, missing)

costplan/planning.py

@@ -0,0 +1,416 @@
+"""Plan what an agent costs from a handful of pilot runs.
+
+Agent costs are lopsided: most runs are cheap, and a few loop, retry or read
+long documents and cost many times more. That makes averages slow to settle
+and makes "what cap should I set?" a question about the tail, not the middle.
+The methods here make no assumption about the shape of the costs: quantiles
+use order statistics, and intervals for averages and ratios use the bootstrap.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from statistics import fmean, stdev
+from typing import Iterable, List, Optional, Sequence, Tuple
+
+from ._stats import (
+    bootstrap_interval,
+    check_level,
+    quantile,
+    quantile_interval,
+    upper_quantile_bound,
+    wilson,
+    z_two_sided,
+)
+
+
+def run_cost(
+    input_tokens: float,
+    output_tokens: float,
+    input_price: float,
+    output_price: float,
+    cached_tokens: float = 0,
+    cached_price: Optional[float] = None,
+) -> float:
+    """Dollar cost of one call or one run from token counts.
+
+    Prices are in dollars per million tokens, as providers list them.
+    ``cached_tokens`` are the part of ``input_tokens`` served from the
+    provider's prompt cache, billed at ``cached_price`` (defaults to the input
+    price). costplan ships no price table, because prices change; take them
+    from your provider's pricing page.
+
+    >>> round(run_cost(12_000, 800, input_price=3.0, output_price=15.0), 4)
+    0.048
+    """
+    for name, value in [("input_tokens", input_tokens), ("output_tokens", output_tokens), ("cached_tokens", cached_tokens)]:
+        if value < 0:
+            raise ValueError(f"{name} must not be negative")
+    if cached_tokens > input_tokens:
+        raise ValueError("cached_tokens are part of input_tokens, so cannot exceed them")
+    if cached_price is None:
+        cached_price = input_price
+    fresh = input_tokens - cached_tokens
+    return (fresh * input_price + cached_tokens * cached_price + output_tokens * output_price) / 1_000_000
+
+
+def _clean(costs: Iterable[float], successes: Optional[Iterable]) -> Tuple[List[float], Optional[List[bool]]]:
+    values = [float(c) for c in costs]
+    if not values:
+        raise ValueError("need at least one run")
+    if any(not math.isfinite(c) or c < 0 for c in values):
+        raise ValueError("costs must be finite and not negative")
+    flags = None
+    if successes is not None:
+        flags = [bool(s) for s in successes]
+        if len(flags) != len(values):
+            raise ValueError("successes needs one value per run, in the same order as costs")
+    return values, flags
+
+
+def _cost_per_success(costs: Sequence[float], flags: Sequence[bool]) -> Optional[float]:
+    wins = sum(flags)
+    return sum(costs) / wins if wins else None
+
+
+def _fmt(x: Optional[float]) -> str:
+    if x is None:
+        return "n/a"
+    return f"${x:,.4f}" if x < 1 else f"${x:,.2f}"
+
+
+@dataclass(frozen=True)
+class CostSummary:
+    n: int
+    mean: float
+    mean_interval: Tuple[float, float]
+    median: float
+    median_interval: Tuple[Optional[float], Optional[float]]
+    p95: float
+    p95_interval: Tuple[Optional[float], Optional[float]]
+    max: float
+    spread: float
+    success_rate: Optional[float]
+    success_interval: Optional[Tuple[float, float]]
+    cost_per_success: Optional[float]
+    cost_per_success_interval: Optional[Tuple[float, float]]
+    confidence: float
+
+    def summary(self) -> str:
+        c = f"{self.confidence:.0%}"
+
+        def rng(pair):
+            lo, hi = pair
+            return f"{_fmt(lo)} to {_fmt(hi)}" if lo is not None and hi is not None else "needs more runs"
+
+        lines = [
+            f"{self.n} runs.",
+            f"  Average cost per run: {_fmt(self.mean)} ({c} interval {rng(self.mean_interval)})",
+            f"  Typical run (median): {_fmt(self.median)} ({c} interval {rng(self.median_interval)})",
+            f"  1 in 20 runs costs more than: {_fmt(self.p95)} ({c} interval {rng(self.p95_interval)})",
+            f"  Most expensive run: {_fmt(self.max)}",
+        ]
+        if self.success_rate is not None:
+            lo, hi = self.success_interval
+            lines.append(f"  Success rate: {self.success_rate:.1%} ({c} interval {lo:.1%} to {hi:.1%})")
+            if self.cost_per_success is None:
+                lines.append("  Cost per successful task: n/a (no successful runs)")
+            else:
+                lines.append(
+                    f"  Cost per successful task: {_fmt(self.cost_per_success)} "
+                    f"({c} interval {rng(self.cost_per_success_interval)})"
+                )
+        return "\n".join(lines)
+
+    def __str__(self) -> str:
+        return self.summary()
+
+
+def summarize(
+    costs: Iterable[float],
+    successes: Optional[Iterable] = None,
+    confidence: float = 0.95,
+    resamples: int = 2000,
+    seed: int = 0,
+) -> CostSummary:
+    """What do these runs cost, and how sure can we be?
+
+    Args:
+        costs: dollar cost of each pilot run.
+        successes: optional pass/fail for each run, to get the success rate
+            and the cost per successful task.
+        confidence: level for every interval.
+        resamples, seed: bootstrap settings. The seed makes results repeatable.
+
+    Intervals for the average and the cost per success use the percentile
+    bootstrap, which is rough below about 30 runs. Quantile intervals make no
+    assumption about the shape of the costs.
+    """
+    check_level(confidence, "confidence")
+    values, flags = _clean(costs, successes)
+    ordered = sorted(values)
+    n = len(values)
+    mean = fmean(values)
+    spread = stdev(values) / mean if n > 1 and mean > 0 else 0.0
+    mean_ci = bootstrap_interval([values], lambda c: fmean(c), confidence, resamples, seed) if n > 1 else (mean, mean)
+    rate = rate_ci = cps = cps_ci = None
+    if flags is not None:
+        wins = sum(flags)
+        rate = wins / n
+        rate_ci = wilson(wins, n, confidence)
+        cps = _cost_per_success(values, flags)
+        if cps is not None and n > 1:
+            cps_ci = bootstrap_interval([values, flags], _cost_per_success, confidence, resamples, seed)
+    return CostSummary(
+        n=n,
+        mean=mean,
+        mean_interval=mean_ci,
+        median=quantile(ordered, 0.5),
+        median_interval=quantile_interval(ordered, 0.5, confidence),
+        p95=quantile(ordered, 0.95),
+        p95_interval=quantile_interval(ordered, 0.95, confidence),
+        max=ordered[-1],
+        spread=spread,
+        success_rate=rate,
+        success_interval=rate_ci,
+        cost_per_success=cps,
+        cost_per_success_interval=cps_ci,
+        confidence=confidence,
+    )
+
+
+# ---------------------------------------------------------------- budget caps
+
+
+def runs_needed_for_cap(stop_rate: float = 0.02, confidence: float = 0.95) -> int:
+    """Pilot runs needed before a cap can promise to stop at most ``stop_rate`` of normal runs.
+
+    With this many runs, setting the cap at the most expensive pilot run
+    keeps the share of normal runs that hit the cap at or below
+    ``stop_rate``, with the given confidence, whatever the cost distribution.
+
+    >>> runs_needed_for_cap(0.02, 0.95)
+    149
+    """
+    check_level(stop_rate, "stop_rate")
+    check_level(confidence, "confidence")
+    return math.ceil(math.log(1 - confidence) / math.log(1 - stop_rate) - 1e-9)
+
+
+def guaranteed_stop_rate(n_runs: int, confidence: float = 0.95) -> float:
+    """The best promise a cap at the most expensive of ``n_runs`` pilot runs can make.
+
+    >>> round(guaranteed_stop_rate(30), 3)
+    0.095
+    """
+    if n_runs < 1:
+        raise ValueError("n_runs must be at least 1")
+    check_level(confidence, "confidence")
+    return 1 - (1 - confidence) ** (1 / n_runs)
+
+
+def _pct_label(p: float) -> str:
+    value = round(p * 100, 1)
+    return f"{value:g}th"
+
+
+@dataclass(frozen=True)
+class CapAdvice:
+    n: int
+    stop_rate: float
+    confidence: float
+    estimate: float
+    safe_cap: Optional[float]
+    runs_needed: int
+    best_promise: float
+    max_cost: float
+
+    def summary(self) -> str:
+        head = (
+            f"Goal: a cap that stops at most {self.stop_rate:.1%} of normal runs "
+            f"({self.confidence:.0%} confidence), from {self.n} pilot runs."
+        )
+        if self.safe_cap is not None:
+            return (
+                f"{head}\n  Recommended cap: {_fmt(self.safe_cap)} per run.\n"
+                f"  Best single guess for the {_pct_label(1 - self.stop_rate)} percentile: {_fmt(self.estimate)}. "
+                "The recommended cap is set higher so the promise holds even if the pilot was lucky."
+            )
+        return (
+            f"{head}\n  Not enough runs to promise that yet: it takes {self.runs_needed}.\n"
+            f"  With {self.n} runs, a cap at the most expensive run ({_fmt(self.max_cost)}) can only promise to stop "
+            f"at most {self.best_promise:.1%} of runs.\n"
+            f"  Best single guess for the {_pct_label(1 - self.stop_rate)} percentile: {_fmt(self.estimate)} "
+            "(a guess, not a promise)."
+        )
+
+    def __str__(self) -> str:
+        return self.summary()
+
+
+def recommend_cap(costs: Iterable[float], stop_rate: float = 0.02, confidence: float = 0.95) -> CapAdvice:
+    """What per-run budget cap to set, so it rarely stops a normal run.
+
+    The cap is an upper confidence bound on the ``1 - stop_rate`` quantile of
+    run cost, from order statistics, so it makes no assumption about the cost
+    distribution. If there are too few runs to make that promise, the advice
+    says how many are needed and what the current pilot can promise instead.
+    """
+    check_level(stop_rate, "stop_rate")
+    check_level(confidence, "confidence")
+    values, _ = _clean(costs, None)
+    ordered = sorted(values)
+    n = len(ordered)
+    return CapAdvice(
+        n=n,
+        stop_rate=stop_rate,
+        confidence=confidence,
+        estimate=quantile(ordered, 1 - stop_rate),
+        safe_cap=upper_quantile_bound(ordered, 1 - stop_rate, confidence),
+        runs_needed=runs_needed_for_cap(stop_rate, confidence),
+        best_promise=guaranteed_stop_rate(n, confidence),
+        max_cost=ordered[-1],
+    )
+
+
+# --------------------------------------------------------- pilot size, spend
+
+
+def pilot_runs_for_average(spread: float, margin: float = 0.10, confidence: float = 0.95) -> int:
+    """Pilot runs needed to pin the average cost to within ``margin`` (relative).
+
+    ``spread`` is the coefficient of variation: standard deviation divided by
+    the average. Agent costs often have a spread of 1 or more, because of the
+    occasional runaway run. Get it from a small pilot with ``summarize``.
+
+    >>> pilot_runs_for_average(spread=1.2, margin=0.10)
+    554
+    """
+    if spread < 0:
+        raise ValueError("spread must not be negative")
+    check_level(margin, "margin")
+    check_level(confidence, "confidence")
+    z = z_two_sided(confidence)
+    return max(2, math.ceil((z * spread / margin) ** 2 - 1e-9))
+
+
+@dataclass(frozen=True)
+class SpendProjection:
+    runs: int
+    expected: float
+    low: float
+    high: float
+    confidence: float
+
+    def summary(self) -> str:
+        return (
+            f"{self.runs:,} runs: about {_fmt(self.expected)} "
+            f"({self.confidence:.0%} range {_fmt(self.low)} to {_fmt(self.high)})."
+        )
+
+    def __str__(self) -> str:
+        return self.summary()
+
+
+def project_spend(costs: Iterable[float], runs: int, confidence: float = 0.95) -> SpendProjection:
+    """Total spend for ``runs`` future runs, given pilot run costs.
+
+    The range includes both run to run variation and the uncertainty in the
+    average learned from the pilot. For large volumes the second one
+    dominates, so a bigger pilot narrows the range more than anything else.
+    """
+    if runs < 1:
+        raise ValueError("runs must be at least 1")
+    check_level(confidence, "confidence")
+    values, _ = _clean(costs, None)
+    n = len(values)
+    if n < 2:
+        raise ValueError("need at least two pilot runs to project spend")
+    mean, sd = fmean(values), stdev(values)
+    se_total = math.sqrt(runs * runs * sd * sd / n + runs * sd * sd)
+    half = z_two_sided(confidence) * se_total
+    expected = mean * runs
+    return SpendProjection(runs, expected, max(0.0, expected - half), expected + half, confidence)
+
+
+# --------------------------------------------------------------- comparison
+
+
+@dataclass(frozen=True)
+class CostComparison:
+    a: str
+    b: str
+    a_cost_per_success: Optional[float]
+    b_cost_per_success: Optional[float]
+    ratio: Optional[float]
+    ratio_interval: Optional[Tuple[float, float]]
+    confidence: float
+    paired: bool = False
+
+    def summary(self) -> str:
+        if self.ratio is None:
+            return f"Cannot compare: {self.a} or {self.b} has no successful runs."
+        lo, hi = self.ratio_interval
+        if hi < 1:
+            verdict = f"{self.b} is reliably cheaper per success."
+        elif lo > 1:
+            verdict = f"{self.b} is reliably more expensive per success."
+        else:
+            verdict = "The difference could be noise."
+        return (
+            f"Cost per successful task: {self.a} {_fmt(self.a_cost_per_success)}, {self.b} {_fmt(self.b_cost_per_success)}.\n"
+            f"  {self.b} costs {self.ratio:.2f} times as much as {self.a} per success "
+            f"({self.confidence:.0%} interval {lo:.2f} to {hi:.2f}, "
+            f"{'same tasks, compared task by task' if self.paired else 'separate pilots'}). {verdict}"
+        )
+
+    def __str__(self) -> str:
+        return self.summary()
+
+
+def compare_cost_per_success(
+    a_costs: Iterable[float],
+    a_successes: Iterable,
+    b_costs: Iterable[float],
+    b_successes: Iterable,
+    names: Tuple[str, str] = ("A", "B"),
+    paired: bool = False,
+    confidence: float = 0.95,
+    resamples: int = 2000,
+    seed: int = 0,
+) -> CostComparison:
+    """Which setup is cheaper per successful task?
+
+    A cheaper model that fails more often can cost more per task that gets
+    done. The interval is a bootstrap of the ratio of costs per success.
+
+    Args:
+        paired: True when both setups ran the same tasks, in the same order.
+            Hard tasks are expensive for both, so comparing task by task
+            removes that variation and gives a much tighter interval from the
+            same number of runs. False when the two pilots ran different tasks.
+    """
+    import random
+
+    check_level(confidence, "confidence")
+    ac, af = _clean(a_costs, a_successes)
+    bc, bf = _clean(b_costs, b_successes)
+    if paired and len(ac) != len(bc):
+        raise ValueError("a paired comparison needs the same tasks for both setups, so the same number of runs")
+    a_cps, b_cps = _cost_per_success(ac, af), _cost_per_success(bc, bf)
+    if a_cps is None or b_cps is None:
+        return CostComparison(names[0], names[1], a_cps, b_cps, None, None, confidence, paired)
+    rng = random.Random(seed)
+    ratios = []
+    for _ in range(resamples):
+        ia = [rng.randrange(len(ac)) for _ in ac]
+        ib = ia if paired else [rng.randrange(len(bc)) for _ in bc]
+        ra = _cost_per_success([ac[i] for i in ia], [af[i] for i in ia])
+        rb = _cost_per_success([bc[i] for i in ib], [bf[i] for i in ib])
+        if ra and rb is not None:
+            ratios.append(rb / ra)
+    ratios.sort()
+    tail = (1 - confidence) / 2
+    interval = (quantile(ratios, tail), quantile(ratios, 1 - tail))
+    return CostComparison(names[0], names[1], a_cps, b_cps, b_cps / a_cps, interval, confidence, paired)

costplan-0.1.0.dist-info/METADATA

@@ -0,0 +1,229 @@
+Metadata-Version: 2.4
+Name: costplan
+Version: 0.1.0
+Summary: How much will my agent cost? Plan AI agent budgets, caps and spend from a handful of pilot runs
+Author: Meenal Asesh
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/ai-with-md/costplan
+Project-URL: Calculator, https://ai-with-md.github.io/costplan/
+Project-URL: Issues, https://github.com/ai-with-md/costplan/issues
+Keywords: llm,ai agents,cost,budget,token cost,llm cost,cost per task,statistics,forecasting
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+# costplan
+
+**How much will my agent cost?** Plan AI agent budgets, caps and spend from a handful of pilot runs, with a [calculator that runs in your browser](https://ai-with-md.github.io/costplan/).
+
+[![PyPI](https://img.shields.io/pypi/v/costplan)](https://pypi.org/project/costplan/)
+[![tests](https://github.com/ai-with-md/costplan/actions/workflows/tests.yml/badge.svg)](https://github.com/ai-with-md/costplan/actions/workflows/tests.yml)
+![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)
+![License: MIT](https://img.shields.io/badge/license-MIT-green)
+
+[![The costplan calculator: run costs, a budget cap and the long tail of expensive runs](https://raw.githubusercontent.com/ai-with-md/costplan/main/docs/calculator.png)](https://ai-with-md.github.io/costplan/)
+
+Agent costs are lopsided. Most runs take a few tool calls and cost a few cents. A few get stuck in a loop, retry, or read a long document, and cost ten times more. In the example pilot above, the typical run costs $0.11 and the most expensive one $1.43. That makes averages slow to settle, and it makes "what budget cap should I set?" a question about the rare expensive runs, not the typical ones.
+
+costplan answers the planning questions from a pilot:
+
+1. What does a run really cost, and what does a **finished task** cost?
+2. What per-run budget cap should I set, so it almost never stops a normal run?
+3. How many pilot runs do I need before I can trust those numbers?
+4. What will it cost at 10,000 runs a day?
+5. Is the cheaper model actually cheaper per finished task?
+
+## The number most people get wrong
+
+To set a budget cap that stops at most 2% of normal runs, and be 95% sure of it, you need **149 pilot runs**. Fewer runs simply cannot rule out a rare expensive run that a tighter cap would cut off. Set the cap at the most expensive pilot run and this is the best you can promise:
+
+| Pilot runs | Cap stops at most |
+|---|---|
+| 20 | 13.9% of normal runs |
+| 30 | 9.5% |
+| 50 | 5.8% |
+| 100 | 3.0% |
+| 149 | 2.0% |
+| 300 | 1.0% |
+
+These numbers hold whatever the shape of your costs. costplan tells you what your current pilot can promise, and how many more runs the promise you want would take.
+
+## Try it
+
+**In the browser:** [ai-with-md.github.io/costplan](https://ai-with-md.github.io/costplan/). Paste your run costs, nothing to install.
+
+**In Python:**
+
+```bash
+pip install costplan
+```
+
+No dependencies beyond the standard library. Python 3.9 or newer.
+
+## Use
+
+### Reading your exports
+
+`load_runs` reads a CSV, JSON or JSON Lines export from your tracing tool or provider and finds the cost, token, success and run columns by their common names (`cost`, `total_cost`, `totalCost`, `calculatedTotalCost`, `prompt_tokens`, `input_tokens`, `usage.input`, `status`, `error`, `trace_id`, `traceId` and others).
+
+```python
+import costplan as cp
+
+runs = cp.load_runs("calls_export.jsonl", group_by="trace_id")
+print(runs.report())
+# Read 318 rows into 40 runs (cost from 'cost', status from 'status', runs from 'trace_id').
+
+costs, successes = runs.costs, runs.successes
+```
+
+- **One row per model call?** Pass `group_by` with the column that names the run (a trace or session id). Calls are added up into runs, and a run counts as failed if any of its calls failed.
+- **One row per run, already totalled?** Load it as it is. Grouping it again would count costs twice.
+- **No cost column, or calls your tool could not price?** Pass `input_price` and `output_price` in dollars per million tokens and costplan prices them from the token columns. costplan ships no price table, because prices change; take them from your provider's pricing page or a list such as [tokencost](https://github.com/AgentOps-AI/tokencost).
+- **Names not found?** Pass them yourself: `cost="spend"`, `success="passed"`, and dotted names for nested JSON such as `"usage.prompt_tokens"`.
+
+The column guesses are tested on files shaped like LangSmith run exports and Langfuse observation exports, using the field names in their documentation. Check `runs.report()` the first time you load a new kind of file.
+
+`run_cost` does the same pricing for a single call:
+
+```python
+cp.run_cost(input_tokens=12_000, output_tokens=800, input_price=3.0, output_price=15.0)
+# 0.048
+```
+
+### 1. What does a run cost?
+
+```python
+print(cp.summarize(costs, successes))
+# 60 runs.
+#   Average cost per run: $0.1577 (95% interval $0.1160 to $0.2167)
+#   Typical run (median): $0.1100 (95% interval $0.0920 to $0.1371)
+#   1 in 20 runs costs more than: $0.3272 (95% interval needs more runs)
+#   Most expensive run: $1.43
+#   Success rate: 85.0% (95% interval 73.9% to 91.9%)
+#   Cost per successful task: $0.1855 (95% interval $0.1308 to $0.2677)
+```
+
+The cost per successful task is the number that matters for a product: failed runs cost money too.
+
+### 2. What cap should I set?
+
+```python
+print(cp.recommend_cap(costs, stop_rate=0.02))
+# Goal: a cap that stops at most 2.0% of normal runs (95% confidence), from 60 pilot runs.
+#   Not enough runs to promise that yet: it takes 149.
+#   With 60 runs, a cap at the most expensive run ($1.43) can only promise to stop at most 4.9% of runs.
+#   Best single guess for the 98th percentile: $0.7737 (a guess, not a promise).
+```
+
+With enough runs, it returns a recommended cap. That cap is an upper confidence bound on the cost percentile, so it holds even if the pilot happened to miss some expensive runs. Enforce it with your agent framework, your own code, or a library such as [AgentBudget](https://github.com/AgentBudget/agentbudget).
+
+### 3. How many pilot runs do I need?
+
+```python
+cp.runs_needed_for_cap(stop_rate=0.02)       # 149
+cp.guaranteed_stop_rate(n_runs=30)           # 0.095
+cp.pilot_runs_for_average(spread=1.0, margin=0.10)   # 385 runs to pin the average within 10%
+```
+
+`spread` is the standard deviation divided by the average. Agent costs often have a spread near 1 or above because of the runaway runs, which is why averages need hundreds of runs to settle.
+
+### 4. What will it cost at scale?
+
+```python
+print(cp.project_spend(costs, runs=300_000))
+# 300,000 runs: about $47,301.75 (95% range $31,765.55 to $62,837.95).
+```
+
+At high volume the run to run variation averages out, and nearly all of the range comes from how well the pilot pinned down the average. A bigger pilot narrows it more than anything else.
+
+### 5. Is the cheaper model cheaper per finished task?
+
+```python
+print(cp.compare_cost_per_success(large_costs, large_ok, small_costs, small_ok, names=("large model", "small model")))
+# Cost per successful task: large model $0.1855, small model $0.2197.
+#   small model costs 1.18 times as much as large model per success (95% interval 0.71 to 2.02, separate pilots). The difference could be noise.
+```
+
+In this example the small model is about 30% cheaper per run, but it fails and loops more often, so it probably costs more per finished task. With 60 runs each, costplan correctly says the difference is not yet certain.
+
+**Run both models on the same tasks if you can.** Hard tasks are expensive for every model, so comparing task by task removes that variation. With `paired=True` the same 60 tasks give an answer that separate pilots cannot:
+
+```python
+cp.compare_cost_per_success(large_costs, large_ok, small_costs, small_ok, names=("large model", "small model"))
+#   small model costs 0.73 times as much as large model per success (95% interval 0.49 to 1.08, separate pilots). The difference could be noise.
+
+cp.compare_cost_per_success(large_costs, large_ok, small_costs, small_ok, names=("large model", "small model"), paired=True)
+#   small model costs 0.73 times as much as large model per success (95% interval 0.61 to 0.88, same tasks, compared task by task). small model is reliably cheaper per success.
+```
+
+In simulation, pairing gives intervals about half as wide from the same number of runs.
+
+## Command line
+
+```bash
+costplan summary runs.csv
+costplan cap     runs.csv --stop-rate 0.02
+costplan project runs.csv --runs 300000
+costplan compare large.csv small.csv --names "large model" "small model"
+costplan compare large.csv small.csv --paired --task-column task_id
+costplan summary calls_export.jsonl --group-by trace_id -v
+costplan pilot   --stop-rate 0.02 --spread 1.0
+```
+
+Files can be CSV, JSON or JSON Lines, with the same automatic column finding as `load_runs`. Use `--cost-column`, `--success-column` and similar options when the guess is wrong, `--input-price` and `--output-price` for files with tokens but no costs, and `-v` to see which columns were used. Try it on the simulated files in [`examples/`](https://github.com/ai-with-md/costplan/tree/main/examples).
+
+## How the numbers are checked
+
+The test suite simulates thousands of pilots with lopsided costs where the truth is known, and checks each promise:
+
+| Promise | Held in simulation |
+|---|---|
+| The recommended cap stops at most 2% of runs | 98.5% of pilots (95% promised) |
+| A cap at the most expensive of 149 runs stops at most 2% | 94.8% |
+| The interval for cost per successful task covers the truth | 93% |
+| The spend projection covers the real bill | 92% |
+| The interval for the average cost covers the truth | 92% |
+| The paired comparison interval covers the true ratio | 96% |
+| Paired vs separate pilots, same number of runs | interval about half as wide |
+
+The caps and quantiles make no assumption about the shape of costs and keep their promise. The intervals for averages run slightly short of 95% on lopsided costs, which is typical for the bootstrap; treat them as a little optimistic, especially below about 30 runs. See [`tests/test_simulation.py`](https://github.com/ai-with-md/costplan/blob/main/tests/test_simulation.py).
+
+## How it compares
+
+- **[AgentBudget](https://github.com/AgentBudget/agentbudget)** and similar tools enforce a dollar limit while an agent runs. costplan tells you what limit to set, and what you will spend. They work well together.
+- **Price calculators and price lists** such as [tokencost](https://github.com/AgentOps-AI/tokencost) cost out a single request. costplan works on whole runs, including the expensive tail, retries and failures.
+- **Tracing tools** (Langfuse, LangSmith and others) record what runs cost. costplan turns those records into planning numbers with honest uncertainty.
+
+## Limits
+
+- The pilot must look like real traffic: the same prompts, tools, models and mix of requests. Change any of them and run a new pilot.
+- Caps and projections assume future runs behave like pilot runs. A new kind of request that triggers long loops will not show up in a pilot that never saw it.
+- The spend projection uses a normal approximation, which is a little narrow for very lopsided costs.
+
+## More tools
+
+Small, free tools for building and checking AI systems:
+
+- **[sturdytools](https://github.com/ai-with-md/sturdytools)**: retries, timeouts, rate limits and circuit breakers for the tools an AI agent calls.
+- **[evalsize](https://github.com/ai-with-md/evalsize)**: how many eval examples you need before a difference between prompts or models means anything.
+
+## Development
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -e ".[dev]"
+pytest
+```
+
+The calculator is a [marimo](https://marimo.io) notebook in [`app/calculator.py`](https://github.com/ai-with-md/costplan/blob/main/app/calculator.py), published to GitHub Pages as a WebAssembly app on every push to `main`.
+
+## License
+
+MIT

costplan-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+costplan/__init__.py,sha256=yDI4XKayQG9O8iG8h09ApV96XpJVn1_Cov-P3LbyYXo,749
+costplan/_stats.py,sha256=UB2QyfcFDTby3iNhTYRehht2tPfghI-yGU3aSDhV-Eo,3683
+costplan/cli.py,sha256=jgQ3HLAHoLR_XKimV-Snwz5aneYiQedkuxQeC2g-tKE,7170
+costplan/loading.py,sha256=Y0offWXr0m42rdob1_8782tnmQV_c5U4cnJ-8yWXrKo,11031
+costplan/planning.py,sha256=DqB77HFo2guXYeYD_HjrScnxbTGZiJLypuu1K5UwabU,15582
+costplan-0.1.0.dist-info/licenses/LICENSE,sha256=HS_Pp1J0NNKF4niTKA1MdskkGwByEasM-oJb8NcpaGE,1069
+costplan-0.1.0.dist-info/METADATA,sha256=3l6bTfiWk-hpRZRMY8P6ga4QdJJ16h4n4fsXEYohDLE,12235
+costplan-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+costplan-0.1.0.dist-info/entry_points.txt,sha256=fC2SC_ksmyycIaDe8HQPOUs2wdKWOthgJOidhsIsnyE,47
+costplan-0.1.0.dist-info/top_level.txt,sha256=s9Jb01a6gxXW-NvYZvOmWABKLMGyWtjuWFhgftzq1P0,9
+costplan-0.1.0.dist-info/RECORD,,

costplan-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

costplan-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+costplan = costplan.cli:main

costplan-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Meenal Asesh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

costplan-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+costplan