pymelvil 0.3.0__py3-none-any.whl

melvil/__init__.py

@@ -0,0 +1,52 @@
+"""melvil: optimized, versioned classifier prompts from labeled examples.
+
+Quickstart::
+
+    import melvil as mv
+
+    examples = mv.load_csv("tickets.csv")                  # text,label columns
+    train, dev = mv.train_dev_split(examples, dev_size=100, seed=0)
+    spec = mv.TaskSpec.from_examples("ticket-intents", examples)
+    cfg = mv.Config(task_model="openai/gpt-4.1-mini",
+                    reflection_model="openai/gpt-4.1", budget="light")
+    artifact = mv.optimize(spec, train, dev, cfg)
+    print(artifact.render())                               # deployable prompt
+    artifact.save("ticket_intents.v1.json")
+"""
+
+from melvil._about import LIBRARY_NAME, __version__
+from melvil.adapter import RoundInfo
+from melvil.artifact import PromptArtifact, render_prompt
+from melvil.config import BUDGET_PRESETS, Config, Features
+from melvil.costs import CostEstimate, estimate_evaluate_cost, estimate_optimize_cost
+from melvil.data import Example, load_csv, load_hf, train_dev_split
+from melvil.evaluate import Report, evaluate, report
+from melvil.optimize import optimize
+from melvil.screen import ScreenResult, screen
+from melvil.taskspec import Label, TaskSpec
+
+__all__ = [
+    "BUDGET_PRESETS",
+    "LIBRARY_NAME",
+    "Config",
+    "CostEstimate",
+    "Example",
+    "Features",
+    "Label",
+    "PromptArtifact",
+    "Report",
+    "RoundInfo",
+    "TaskSpec",
+    "__version__",
+    "estimate_evaluate_cost",
+    "estimate_optimize_cost",
+    "evaluate",
+    "load_csv",
+    "load_hf",
+    "optimize",
+    "render_prompt",
+    "report",
+    "screen",
+    "ScreenResult",
+    "train_dev_split",
+]

melvil/_about.py

@@ -0,0 +1,6 @@
+"""Library identity. The name lives HERE and nowhere else in code — a rename is
+this constant plus the package directory and the `[project]` table in
+pyproject.toml (one commit)."""
+
+LIBRARY_NAME = "melvil"
+__version__ = "0.3.0"

melvil/adapter.py

@@ -0,0 +1,160 @@
+"""The gepa `GEPAAdapter` for classification: builds the prompt from candidate
+components, evaluates batches with the task LM, maintains confusion state on
+full dev evaluations, and produces reflective datasets per component."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from gepa.core.adapter import EvaluationBatch
+
+from melvil.artifact import BLOB_COMPONENT, LABEL_PREFIX, render_prompt
+from melvil.config import Config
+from melvil.data import Example
+from melvil.features.confusion import ConfusionState, render_confusion_block
+from melvil.program import Prediction, classify_batch
+from melvil.proposer import ClassificationProposer
+from melvil.taskspec import TaskSpec
+
+
+@dataclass
+class RoundInfo:
+    """Passed to the `on_round` callback after every full dev evaluation."""
+
+    round: int
+    dev_score: float
+    best_dev_score: float
+    metric_calls_spent: int
+    cost_usd: float
+    top_confusions: list[tuple[str, str, int]] = field(default_factory=list)
+
+
+class ClassificationAdapter:
+    """One instance per optimization run."""
+
+    def __init__(
+        self,
+        spec: TaskSpec,
+        task_lm: Any,
+        config: Config,
+        valset: list[Example],
+        trace_path=None,
+        on_round: Callable[[RoundInfo], None] | None = None,
+        cost_fn: Callable[[], float] | None = None,
+    ):
+        self.spec = spec
+        self.task_lm = task_lm
+        self.config = config
+        self.label_names = spec.label_names
+        self.confusion = ConfusionState()
+        self.on_round = on_round
+        self.cost_fn = cost_fn or (lambda: 0.0)
+        self._val_texts = tuple(e.text for e in valset)
+        self._metric_calls = 0
+        self._best_dev = 0.0
+        self._round = 0
+
+        feats = config.features
+        if feats.codebook or feats.confusion_reflection:
+            self.proposer: ClassificationProposer | None = ClassificationProposer(
+                render_full_prompt=lambda cand: render_prompt(cand, self.label_names),
+                confusion_block=(
+                    (lambda: render_confusion_block(self.confusion))
+                    if feats.confusion_reflection
+                    else (lambda: "")
+                ),
+                trace_path=trace_path,
+            )
+        else:
+            self.proposer = None
+
+    # gepa checks `adapter.propose_new_texts is not None`
+    @property
+    def propose_new_texts(self):
+        return self.proposer
+
+    # ------------------------------------------------------------- evaluate
+    def evaluate(
+        self, batch: list[Example], candidate: dict[str, str], capture_traces: bool = False
+    ) -> EvaluationBatch:
+        rendered = render_prompt(candidate, self.label_names)
+        preds = classify_batch(
+            self.task_lm, rendered, batch, self.label_names, self.config.num_threads
+        )
+        self._metric_calls += len(batch)
+        scores = [1.0 if p.correct else 0.0 for p in preds]
+
+        if self._is_full_val(batch):
+            self.confusion.update(preds)
+            self._round += 1
+            dev_score = sum(scores) / len(scores)
+            self._best_dev = max(self._best_dev, dev_score)
+            if self.on_round:
+                self.on_round(
+                    RoundInfo(
+                        round=self._round,
+                        dev_score=dev_score,
+                        best_dev_score=self._best_dev,
+                        metric_calls_spent=self._metric_calls,
+                        cost_usd=self.cost_fn(),
+                        top_confusions=list(self.confusion.last_pairs[:3]),
+                    )
+                )
+
+        trajectories = None
+        if capture_traces:
+            trajectories = [
+                {"example": ex, "prediction": p} for ex, p in zip(batch, preds, strict=True)
+            ]
+        return EvaluationBatch(
+            outputs=[p.predicted for p in preds], scores=scores, trajectories=trajectories
+        )
+
+    def _is_full_val(self, batch: list[Example]) -> bool:
+        return len(batch) == len(self._val_texts) and all(
+            e.text == t for e, t in zip(batch, self._val_texts, strict=True)
+        )
+
+    # ------------------------------------------------- reflective dataset
+    def make_reflective_dataset(
+        self,
+        candidate: dict[str, str],
+        eval_batch: EvaluationBatch,
+        components_to_update: list[str],
+    ) -> dict[str, list[dict[str, Any]]]:
+        preds: list[Prediction] = [t["prediction"] for t in (eval_batch.trajectories or [])]
+        out: dict[str, list[dict[str, Any]]] = {}
+        for comp in components_to_update:
+            if comp.startswith(LABEL_PREFIX):
+                label = comp[len(LABEL_PREFIX):]
+                relevant = [p for p in preds if not p.correct and label in (p.gold, p.predicted)]
+                # pad with confusion-state examples for this label if the minibatch is clean
+                if not relevant:
+                    relevant = [
+                        p for p in self.confusion.last_preds
+                        if not p.correct and label in (p.gold, p.predicted)
+                    ][:3]
+            else:
+                relevant = [p for p in preds if not p.correct]
+            if not relevant:  # nothing wrong -> show the minibatch as-is
+                relevant = preds
+            records = [
+                {
+                    "Inputs": {"text": p.text},
+                    "Generated Outputs": {"response": p.raw},
+                    "Feedback": p.feedback,
+                }
+                for p in relevant[:6]
+            ]
+            if records:
+                out[comp] = records
+        if not out:
+            raise ValueError("no reflective examples available for any component")
+        return out
+
+    # convenience for blob mode default proposer compatibility
+    @staticmethod
+    def blob_candidate(text: str) -> dict[str, str]:
+        return {BLOB_COMPONENT: text}

melvil/artifact.py

@@ -0,0 +1,194 @@
+"""PromptArtifact: the versioned output of an optimization run.
+
+An artifact is a JSON document holding the evolved prompt *components*, the
+models/budget/scores/config that produced them, and lineage. `render()`
+assembles the deployable prompt string — the exact prompt that was executed
+during optimization, not a reconstruction.
+
+Component naming convention:
+- codebook mode: ``task_instruction``, ``label::<name>`` (one per label),
+  ``boundary_rules``
+- blob mode (codebook off): a single ``instruction`` component
+"""
+
+from __future__ import annotations
+
+import difflib
+import hashlib
+import json
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from melvil._about import LIBRARY_NAME, __version__
+
+SCHEMA_VERSION = 1
+
+TASK_COMPONENT = "task_instruction"
+BLOB_COMPONENT = "instruction"
+BOUNDARY_COMPONENT = "boundary_rules"
+LABEL_PREFIX = "label::"
+
+#: Fixed output contract appended to every rendered prompt. It is NOT an
+#: evolvable component, so the optimizer can never break the output format.
+OUTPUT_CONTRACT = "Respond with exactly one category name from the list above and nothing else."
+
+
+def label_component(name: str) -> str:
+    return f"{LABEL_PREFIX}{name}"
+
+
+def render_prompt(
+    components: dict[str, str],
+    label_names: list[str],
+    exemplars: list[dict[str, str]] | None = None,
+) -> str:
+    """Assemble the deployable system prompt from components (+ optional mined
+    few-shot exemplars). Used identically at optimize time and via
+    `PromptArtifact.render()`."""
+    parts: list[str] = []
+    if BLOB_COMPONENT in components:  # blob mode
+        parts.append(components[BLOB_COMPONENT].strip())
+    else:
+        parts.append(components.get(TASK_COMPONENT, "").strip())
+        defs = []
+        for name in label_names:
+            desc = components.get(label_component(name), "").strip()
+            defs.append(f"- {name}: {desc}" if desc else f"- {name}")
+        parts.append("Categories:\n" + "\n".join(defs))
+        boundary = components.get(BOUNDARY_COMPONENT, "").strip()
+        if boundary:
+            parts.append("Boundary rules:\n" + boundary)
+    if exemplars:
+        shot = []
+        for ex in exemplars:
+            shot.append(f"Text: {ex['text']}\nLabel: {ex['label']}")
+        parts.append("Examples:\n\n" + "\n\n".join(shot))
+    parts.append(OUTPUT_CONTRACT)
+    return "\n\n".join(p for p in parts if p)
+
+
+@dataclass
+class PromptArtifact:
+    task_name: str
+    components: dict[str, str]
+    label_names: list[str]
+    exemplars: list[dict[str, str]] = field(default_factory=list)
+    models: dict[str, str] = field(default_factory=dict)
+    budget: dict[str, Any] = field(default_factory=dict)  # metric_calls_spent, cost_usd, ...
+    scores: dict[str, Any] = field(default_factory=dict)  # dev_accuracy, dev_macro_f1, ...
+    confusion: dict[str, Any] | None = None  # {"labels": [...], "matrix": [[...]]} on dev
+    curve: list[dict[str, Any]] = field(default_factory=list)  # metric_calls -> best dev score
+    config: dict[str, Any] = field(default_factory=dict)
+    config_hash: str = ""
+    parent_id: str | None = None
+    artifact_id: str = ""
+    created_at: str = ""
+    library: dict[str, str] = field(
+        default_factory=lambda: {"name": LIBRARY_NAME, "version": __version__}
+    )
+    schema_version: int = SCHEMA_VERSION
+    notes: str = ""
+
+    def __post_init__(self) -> None:
+        if not self.created_at:
+            self.created_at = time.strftime("%Y-%m-%dT%H:%M:%S%z")
+        if not self.artifact_id:
+            payload = json.dumps(
+                [self.task_name, self.components, self.exemplars, self.created_at,
+                 self.config_hash],
+                sort_keys=True,
+            )
+            self.artifact_id = hashlib.sha256(payload.encode()).hexdigest()[:12]
+
+    # ------------------------------------------------------------- behavior
+    def render(self) -> str:
+        """The deployable prompt string."""
+        return render_prompt(self.components, self.label_names, self.exemplars)
+
+    def diff(self, other: PromptArtifact) -> str:
+        """Human-readable comparison: per-component text diff + score deltas."""
+        lines: list[str] = [f"artifact {other.artifact_id} -> {self.artifact_id}"]
+        for key in ("dev_accuracy", "dev_macro_f1", "test_accuracy"):
+            a, b = other.scores.get(key), self.scores.get(key)
+            if a is not None or b is not None:
+                def fmt(v):
+                    return "—" if v is None else f"{v:.3f}"
+
+                delta = f" ({b - a:+.3f})" if (a is not None and b is not None) else ""
+                lines.append(f"  {key}: {fmt(a)} -> {fmt(b)}{delta}")
+        all_comps = sorted(set(self.components) | set(other.components))
+        for comp in all_comps:
+            old = other.components.get(comp, "")
+            new = self.components.get(comp, "")
+            if old == new:
+                continue
+            lines.append(f"\n## {comp}")
+            diff = difflib.unified_diff(
+                old.splitlines(), new.splitlines(), lineterm="",
+                fromfile=f"{comp}@{other.artifact_id}", tofile=f"{comp}@{self.artifact_id}",
+            )
+            lines.extend(list(diff)[2:] or ["(whitespace-only change)"])
+        if len(all_comps) and len(lines) == 1:
+            lines.append("  (no component changes)")
+        ex_a, ex_b = len(other.exemplars), len(self.exemplars)
+        if ex_a != ex_b:
+            lines.append(f"\nexemplars: {ex_a} -> {ex_b}")
+        return "\n".join(lines)
+
+    # ---------------------------------------------------------- persistence
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "artifact_id": self.artifact_id,
+            "parent_id": self.parent_id,
+            "task_name": self.task_name,
+            "created_at": self.created_at,
+            "library": self.library,
+            "models": self.models,
+            "budget": self.budget,
+            "scores": self.scores,
+            "config_hash": self.config_hash,
+            "config": self.config,
+            "label_names": self.label_names,
+            "components": self.components,
+            "exemplars": self.exemplars,
+            "confusion": self.confusion,
+            "curve": self.curve,
+            "notes": self.notes,
+        }
+
+    def save(self, path: str | Path) -> Path:
+        path = Path(path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(self.to_dict(), indent=1, ensure_ascii=False))
+        return path
+
+    @classmethod
+    def load(cls, path: str | Path) -> PromptArtifact:
+        d = json.loads(Path(path).read_text())
+        if d.get("schema_version", 0) > SCHEMA_VERSION:
+            raise ValueError(
+                f"artifact schema {d['schema_version']} is newer than this "
+                f"{LIBRARY_NAME} ({SCHEMA_VERSION}); please upgrade"
+            )
+        return cls(
+            task_name=d["task_name"],
+            components=d["components"],
+            label_names=d["label_names"],
+            exemplars=d.get("exemplars", []),
+            models=d.get("models", {}),
+            budget=d.get("budget", {}),
+            scores=d.get("scores", {}),
+            confusion=d.get("confusion"),
+            curve=d.get("curve", []),
+            config=d.get("config", {}),
+            config_hash=d.get("config_hash", ""),
+            parent_id=d.get("parent_id"),
+            artifact_id=d["artifact_id"],
+            created_at=d.get("created_at", ""),
+            library=d.get("library", {}),
+            schema_version=d.get("schema_version", SCHEMA_VERSION),
+            notes=d.get("notes", ""),
+        )

melvil/config.py

@@ -0,0 +1,114 @@
+"""Run configuration: models, budget presets, feature toggles, seeds.
+
+Plain dataclasses — no YAML required, but `Config.from_yaml()` exists for teams
+that want files.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+#: Budget presets in metric calls (one metric call = one task-model
+#: classification). "light" matches the dspy auto="light" budget for a
+#: single-predictor program with a 100-example dev set (measured in the pilot).
+BUDGET_PRESETS: dict[str, int] = {"light": 800, "medium": 2000, "heavy": 5000}
+
+
+@dataclass
+class Features:
+    """The classification layer. Each feature is independently toggleable
+    (benchmark ablations rely on this).
+
+    - codebook: structure the prompt as named components (task instruction,
+      one definition per label, boundary rules) and let GEPA evolve them
+      per-component. Off = single free-text instruction blob (vanilla GEPA).
+    - confusion_reflection: compute the dev confusion matrix as the run
+      progresses, target the most-confused labels' components for update, and
+      show the reflection LM the top confused pairs with concrete misclassified
+      examples.
+    - hard_example_mining: persistently misclassified dev examples become
+      candidate few-shot exemplars, selected to cover the top confused
+      boundaries; kept only if they don't hurt dev accuracy (one extra dev
+      eval, reserved out of the budget).
+    """
+
+    codebook: bool = False
+    confusion_reflection: bool = False
+    hard_example_mining: bool = False
+
+    @classmethod
+    def none(cls) -> Features:
+        """Vanilla GEPA: all classification features off. This is the default —
+        the strongest configuration in our pre-registered confirmation benchmarks
+        at light budgets (see benchmarks/RESULTS.md)."""
+        return cls(codebook=False, confusion_reflection=False, hard_example_mining=False)
+
+    @classmethod
+    def all(cls) -> Features:
+        """The full classification layer. In confirmation benchmarks it trailed
+        the default at light budgets; consider it for large taxonomies (where the
+        fixed confusion selector was directionally positive) or higher budgets
+        (untested — see benchmarks/RESULTS.md)."""
+        return cls(codebook=True, confusion_reflection=True, hard_example_mining=True)
+
+
+@dataclass
+class Config:
+    """Everything `optimize()` needs. `budget` is a preset name or an explicit
+    metric-call cap."""
+
+    task_model: str
+    reflection_model: str
+    budget: int | str = "light"
+    seed: int = 0
+    features: Features = field(default_factory=Features)
+    num_threads: int = 8
+    run_dir: str | None = None
+    task_temperature: float = 0.0
+    task_max_tokens: int = 40
+    reflection_temperature: float = 1.0
+    reflection_max_tokens: int = 8000
+    reflection_minibatch_size: int = 3
+    max_exemplars: int = 6
+    mining_llm_screen: bool = True
+    prices: dict[str, tuple[float, float]] = field(default_factory=dict)  # $/M in, $/M out
+
+    @property
+    def metric_calls(self) -> int:
+        if isinstance(self.budget, int):
+            return self.budget
+        try:
+            return BUDGET_PRESETS[self.budget]
+        except KeyError:
+            raise ValueError(
+                f"budget must be an int or one of {sorted(BUDGET_PRESETS)}; got {self.budget!r}"
+            ) from None
+
+    def to_dict(self) -> dict[str, Any]:
+        d = asdict(self)
+        d["metric_calls"] = self.metric_calls
+        return d
+
+    @property
+    def config_hash(self) -> str:
+        """Stable hash of everything that affects the optimization result
+        (run_dir and thread count excluded)."""
+        d = self.to_dict()
+        d.pop("run_dir", None)
+        d.pop("num_threads", None)
+        return hashlib.sha256(json.dumps(d, sort_keys=True).encode()).hexdigest()[:12]
+
+    @classmethod
+    def from_yaml(cls, path: str | Path) -> Config:
+        import yaml
+
+        d = yaml.safe_load(Path(path).read_text())
+        if "features" in d and isinstance(d["features"], dict):
+            d["features"] = Features(**d["features"])
+        if "prices" in d:
+            d["prices"] = {k: tuple(v) for k, v in d["prices"].items()}
+        return cls(**d)

melvil/costs.py

@@ -0,0 +1,125 @@
+"""Cost accounting (measured, from lm.history — pilot-proven) and dry-run
+estimation (upper bound, printed before any paid run)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+#: USD per 1M tokens: (input, output). Extend/override via Config.prices.
+DEFAULT_PRICES: dict[str, tuple[float, float]] = {
+    "openai/gpt-4.1-mini": (0.40, 1.60),
+    "openai/gpt-4.1": (2.00, 8.00),
+    "openai/gpt-4.1-nano": (0.10, 0.40),
+    "openai/gpt-4o-mini": (0.15, 0.60),
+    "anthropic/claude-haiku-4-5-20251001": (1.00, 5.00),
+    "anthropic/claude-sonnet-4-5-20250929": (3.00, 15.00),
+    "meta-llama/llama-3.1-8b-instruct": (0.02, 0.03),
+}
+
+
+def price_for(
+    model: str, prices: dict[str, tuple[float, float]] | None = None
+) -> tuple[float, float]:
+    table = {**DEFAULT_PRICES, **(prices or {})}
+    if model in table:
+        return table[model]
+    # tolerate provider prefixes, e.g. "openrouter/openai/gpt-4.1"
+    for key, val in table.items():
+        if model.endswith(key) or key.endswith(model):
+            return val
+    raise KeyError(
+        f"no price known for model {model!r}; "
+        f"pass Config(prices={{{model!r}: (in_per_M, out_per_M)}})"
+    )
+
+
+def lm_usage(lm: Any, since: int = 0) -> dict[str, Any]:
+    """Sum real (non-cache-hit) usage from lm.history[since:]. Thread-safe by
+    construction (history is per-instance and append-only); includes provider-
+    reported per-call cost when available (OpenRouter reports it)."""
+    pt = ct = calls = hits = 0
+    cost = 0.0
+    for e in lm.history[since:]:
+        u = e.get("usage") or {}
+        if not u.get("total_tokens"):
+            hits += 1
+            continue
+        calls += 1
+        pt += u.get("prompt_tokens") or 0
+        ct += u.get("completion_tokens") or 0
+        cost += u.get("cost") or e.get("cost") or 0.0
+    return {
+        "model": getattr(lm, "model", "?"),
+        "prompt_tokens": pt,
+        "completion_tokens": ct,
+        "calls": calls,
+        "cache_hits": hits,
+        "cost_usd": round(float(cost), 6),
+    }
+
+
+def usage_cost_usd(
+    usage: dict[str, Any], prices: dict[str, tuple[float, float]] | None = None
+) -> float:
+    """Cost of a usage record; prefers provider-reported cost, falls back to the
+    price table."""
+    if usage.get("cost_usd"):
+        return float(usage["cost_usd"])
+    try:
+        pin, pout = price_for(usage.get("model", ""), prices)
+    except KeyError:
+        return 0.0
+    return (usage.get("prompt_tokens", 0) * pin + usage.get("completion_tokens", 0) * pout) / 1e6
+
+
+def _tokens(text: str) -> int:
+    return max(1, len(text) // 4)  # chars/4 heuristic
+
+
+@dataclass
+class CostEstimate:
+    total_usd: float
+    lines: list[str] = field(default_factory=list)
+
+    def __str__(self) -> str:
+        body = "\n".join(f"  {ln}" for ln in self.lines)
+        return f"{body}\n  ESTIMATED TOTAL (upper bound): ${self.total_usd:.2f}"
+
+
+def estimate_optimize_cost(spec, train, dev, config) -> CostEstimate:
+    """Upper-bound dry-run estimate for one `optimize()` call. Assumptions
+    (calibrated on the pilot): the evolved prompt is ~3x the seed prompt;
+    ~1 reflection per 60 metric calls with ~3.5k in / 700 out tokens; caching
+    is ignored (it only lowers real spend)."""
+    from melvil.artifact import render_prompt
+    from melvil.optimize import seed_candidate_for
+
+    candidate = seed_candidate_for(spec)
+    seed_prompt = render_prompt(candidate, spec.label_names)
+    avg_text = sum(_tokens(e.text) for e in (train + dev)[:200]) / max(1, len((train + dev)[:200]))
+    per_call_in = 3 * _tokens(seed_prompt) + avg_text + 20
+    per_call_out = 8
+    n_calls = config.metric_calls
+    tin, tout = price_for(config.task_model, config.prices)
+    rin, rout = price_for(config.reflection_model, config.prices)
+    task_usd = n_calls * (per_call_in * tin + per_call_out * tout) / 1e6
+    n_refl = max(4, n_calls // 60)
+    refl_usd = n_refl * (3500 * rin + 700 * rout) / 1e6
+    total = task_usd + refl_usd
+    return CostEstimate(
+        total_usd=total,
+        lines=[
+            f"task model {config.task_model}: {n_calls} calls x "
+            f"~{per_call_in:.0f} in / {per_call_out} out tok = ${task_usd:.2f}",
+            f"reflection {config.reflection_model}: ~{n_refl} calls x "
+            f"~3.5k/700 tok = ${refl_usd:.2f}",
+        ],
+    )
+
+
+def estimate_evaluate_cost(artifact, data, model: str, prices=None) -> CostEstimate:
+    per_call_in = _tokens(artifact.render()) + 40
+    pin, pout = price_for(model, prices)
+    usd = len(data) * (per_call_in * pin + 8 * pout) / 1e6
+    return CostEstimate(usd, [f"{model}: {len(data)} calls x ~{per_call_in} in tok = ${usd:.2f}"])