itemeval 0.1.0__py3-none-any.whl

itemeval/__init__.py

@@ -0,0 +1,60 @@
+"""itemeval: item-level LLM evaluation over any API, with built-in budget control.
+
+Two equivalent ways to drive a study:
+
+CLI        itemeval estimate|generate|grade|export|status CONFIG
+Python     cfg  = load_config("configs/my_study.yaml")
+           prep = prepare_study(cfg)
+           estimate_study(prep)          # projected $ per stage
+           run_generate(prep)            # stage 1 -> solutions store
+           run_grade(prep)               # stage 2 -> gradings store
+           export_study(cfg)             # long-format parquet + CSV + ledger
+           build_status(cfg, prep)       # grid completion report
+
+The Python pipeline functions do NOT apply the budget confirmation gate (a
+CLI feature) — compare `estimate_study(...)` totals against your own
+threshold before paid runs.
+"""
+
+from importlib import import_module
+from importlib.metadata import version
+
+from itemeval._config import ExperimentConfig, load_config
+from itemeval._item import Item
+
+__version__ = version("itemeval")
+
+# Pipeline functions resolve lazily (PEP 562) so `import itemeval` stays
+# light: eager imports here would pull inspect_ai/pandas into every CLI start.
+_LAZY = {
+    "prepare_study": ("itemeval._prepare", "prepare_study"),
+    "estimate_study": ("itemeval.budget._estimator", "estimate_study"),
+    "run_generate": ("itemeval.generate._run", "run_generate"),
+    "run_grade": ("itemeval.grade._run", "run_grade"),
+    "export_study": ("itemeval.store._export", "export_study"),
+    "build_status": ("itemeval._status", "build_status"),
+}
+
+__all__ = [
+    "ExperimentConfig",
+    "Item",
+    "__version__",
+    "build_status",
+    "estimate_study",
+    "export_study",
+    "load_config",
+    "prepare_study",
+    "run_generate",
+    "run_grade",
+]
+
+
+def __getattr__(name: str):
+    if name in _LAZY:
+        module_name, attr = _LAZY[name]
+        return getattr(import_module(module_name), attr)
+    raise AttributeError(f"module 'itemeval' has no attribute {name!r}")
+
+
+def __dir__() -> "list[str]":
+    return sorted(__all__)

itemeval/_builtin/config.yaml

@@ -0,0 +1,48 @@
+# itemeval study config — generated by `itemeval init`.
+# Runs free out of the box on the mock provider; see the numbered steps to adapt it.
+study: my_study
+
+# Inputs (prompts/rubrics) resolve relative to THIS config file's directory.
+# Outputs (the studies/ folder) are written under your current working directory.
+# Both anchors are overridable — see docs/wiki/Configuration.md.
+
+benchmark:
+  adapter: hf
+  datasets:
+    # step 1: point this at your HuggingFace dataset and pin a revision.
+    - id: MathArena/usamo_2025
+      revision: 0a2c60f2249e07b8ee76c942bca4f5f87aa959df
+      split: train
+  mapping:
+    # step 1: adjust these to your dataset's column names.
+    id: problem_idx
+    input: problem
+    target: sample_solution
+    grading_scheme: grading_scheme
+    metadata: [points]
+
+solvers:
+  # step 4: replace mockllm/* with real inspect model ids (openai/..., anthropic/..., openrouter/...).
+  models: [mockllm/solver-a, mockllm/solver-b, mockllm/solver-c]
+  temperature: 0.7
+  max_tokens: 1024
+
+facets:
+  # `builtin:` templates ship with itemeval. Run `itemeval init --with-templates`
+  # to copy them locally as editable starters, or write your own under prompts/solver/.
+  prompt: [builtin:minimal, builtin:standard]
+  grader: [mock_judge]
+  rubric: [builtin:standard]
+  replications: 2
+
+graders:
+  mock_judge:
+    model: mockllm/judge     # step 4: swap in a real judge model id
+    max_tokens: 512
+
+crossing: full
+
+budget:
+  policy: dev                # dev: first 2 items only; switch to full-interactive / full-batch when ready
+  confirm_above_usd: 5
+  batch: auto

itemeval/_builtin/prompts/solver/minimal.md

@@ -0,0 +1,4 @@
+Solve the following problem. Show your reasoning, then state your final answer
+on a line starting with "ANSWER:".
+
+{input}

itemeval/_builtin/prompts/solver/standard.md

@@ -0,0 +1,6 @@
+You are a careful, expert problem solver. Read the problem below and work
+through it with a complete, rigorous argument. Show all of your reasoning. End
+with a line starting with "ANSWER:" giving your final answer.
+
+Problem:
+{input}

itemeval/_builtin/rubrics/standard.md

@@ -0,0 +1,17 @@
+You are grading a candidate solution against a reference solution and a grading
+scheme.
+
+Problem:
+{input}
+
+Grading scheme:
+{grading_scheme}
+
+Reference solution:
+{target}
+
+Candidate solution:
+{solution}
+
+Evaluate the candidate solution against the grading scheme. Award a numeric
+score according to the scheme.

itemeval/_config.py

@@ -0,0 +1,252 @@
+"""Experiment config schema and YAML loader.
+
+YAML *shape* validation happens at load; *reference* resolution (template
+files exist, grader names defined) is deferred to prepare/grid-expansion so
+the README sketch validates as-is.
+"""
+
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    PrivateAttr,
+    ValidationError,
+    field_validator,
+    model_validator,
+)
+
+from itemeval._errors import ConfigError
+from itemeval._util import sha256_hex
+
+NAME_RE = r"^[A-Za-z0-9][A-Za-z0-9._-]*$"
+STUDY_RE = r"^[a-z0-9][a-z0-9_-]{0,63}$"
+
+ReasoningEffort = Literal["none", "minimal", "low", "medium", "high", "xhigh", "max"]
+
+# What to do with a *completed* generation that produced no gradable text
+# (empty/blank `solution`, no API error — e.g. a reasoning model whose token
+# budget was spent entirely on hidden reasoning). Distinct from API errors
+# (always re-attempted) and parse failures (always final).
+#   skip  — exclude from grading, but report the count + stop reasons (default)
+#   rerun — also treat as not-done in generate, so a subsequent `generate`
+#           re-attempts them (raise max_tokens / lower reasoning effort first;
+#           an identical request will hit the response cache and stay empty)
+#   grade — send to the judge as-is (an empty answer, typically scored low)
+EmptySolutionPolicy = Literal["skip", "rerun", "grade"]
+
+
+class DatasetSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    revision: str | None = None  # branch/tag/SHA; None -> lock file / resolve at first run
+    split: str = "train"
+    name: str | None = None  # HF config name
+    limit: int | None = Field(default=None, ge=1)  # first N rows, no shuffle
+
+
+class MappingSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    input: str
+    target: str | None = None
+    id: str | None = None  # record column -> Item.id (else row index)
+    grading_scheme: str | None = None
+    metadata: list[str] = Field(default_factory=list)
+
+
+class BenchmarkConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    adapter: Literal["hf"]
+    datasets: list[DatasetSpec] = Field(min_length=1)
+    mapping: MappingSpec
+
+
+class SolversConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    models: list[str] = Field(min_length=1)
+    temperature: float | None = Field(default=None, ge=0.0, le=2.0)
+    max_tokens: int | None = Field(default=None, ge=1)
+    top_p: float | None = Field(default=None, gt=0.0, le=1.0)
+    seed: int | None = None  # recorded; only some providers honor it
+    on_empty: EmptySolutionPolicy = "skip"  # handling of empty (no-error) solutions
+
+    @field_validator("models")
+    @classmethod
+    def _unique_models(cls, v: list[str]) -> list[str]:
+        if len(set(v)) != len(v):
+            raise ValueError("solvers.models must be unique")
+        return v
+
+
+class ModelConfigFacet(BaseModel):
+    """One model-config grid cell (sampling overrides + thinking/reasoning toggle)."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str = Field(pattern=NAME_RE)
+    temperature: float | None = Field(default=None, ge=0.0, le=2.0)
+    max_tokens: int | None = Field(default=None, ge=1)
+    top_p: float | None = Field(default=None, gt=0.0, le=1.0)
+    reasoning_effort: ReasoningEffort | None = None  # OpenAI-style
+    reasoning_tokens: int | None = Field(default=None, ge=1)  # Anthropic extended thinking
+
+
+class FacetsConfig(BaseModel):
+    # `model_config` is reserved on pydantic models; the facet list is stored as
+    # `model_config_facet` with alias "model_config" (the YAML key).
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    prompt: list[str] = Field(default_factory=lambda: ["builtin:standard"], min_length=1)
+    grader: list[str] = Field(default_factory=list)
+    rubric: list[str] = Field(default_factory=lambda: ["builtin:standard"], min_length=1)
+    scorer: Literal["exact_match", "multiple_choice", "numeric"] | None = None
+    replications: int = Field(default=1, ge=1)
+    model_config_facet: list[ModelConfigFacet] = Field(
+        default_factory=lambda: [ModelConfigFacet(name="default")],
+        alias="model_config",
+        min_length=1,
+    )
+
+    @model_validator(mode="after")
+    def _check(self) -> "FacetsConfig":
+        if not self.grader and self.scorer is None:
+            raise ValueError("facets must declare at least one of grader / scorer")
+        names = [m.name for m in self.model_config_facet]
+        if len(set(names)) != len(names):
+            raise ValueError("facets.model_config names must be unique")
+        for field in ("prompt", "grader", "rubric"):
+            vals = getattr(self, field)
+            if len(set(vals)) != len(vals):
+                raise ValueError(f"facets.{field} entries must be unique")
+        return self
+
+
+class GraderSpec(BaseModel):
+    """Judge model spec. Temperature is pinned to 0.0 in v0.1 (ROADMAP M3)."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    model: str
+    max_tokens: int | None = Field(default=2048, ge=1)
+    reasoning_effort: ReasoningEffort | None = None
+
+
+class BudgetConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    policy: Literal["dev", "full-interactive", "full-batch"] = "dev"
+    confirm_above_usd: float = Field(default=5.0, ge=0.0)
+    batch: bool | int | Literal["auto"] = "auto"
+    max_usd: float | None = Field(default=None, gt=0.0)  # hard cap, never overridable
+    dev_items: int = Field(default=2, ge=1)  # dev preset: first N items
+    dev_replications: int | None = Field(default=None, ge=1)  # None = keep config reps
+    pricing_path: str | None = None
+
+
+class ExperimentConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    study: str = Field(pattern=STUDY_RE)
+    output_dir: str = "studies"  # outputs: resolved relative to work_dir (CWD)
+    prompts_dir: str = (
+        "prompts"  # solver templates: <prompts_dir>/solver/<name>.md (relative to config_dir)
+    )
+    rubrics_dir: str = (
+        "rubrics"  # rubric templates: <rubrics_dir>/<name>.md (relative to config_dir)
+    )
+    cache: bool = True  # inspect local response cache, both stages
+    benchmark: BenchmarkConfig
+    solvers: SolversConfig
+    facets: FacetsConfig
+    graders: dict[str, GraderSpec] = Field(default_factory=dict)
+    crossing: Literal["full"] = "full"
+    budget: BudgetConfig = Field(default_factory=BudgetConfig)
+
+    # Two resolution anchors (see docs/wiki/Configuration.md):
+    #   config_dir — the loaded YAML's directory; anchors INPUTS (prompts/rubrics/pricing).
+    #                None for in-memory configs, which then anchor inputs to work_dir.
+    #   work_dir   — defaults to CWD; anchors OUTPUTS (the study directory). Never the package.
+    _config_dir: Path | None = PrivateAttr(default=None)
+    _work_dir: Path = PrivateAttr(default_factory=Path.cwd)
+    _config_path: Path | None = PrivateAttr(default=None)
+    _config_sha256: str | None = PrivateAttr(default=None)
+
+    @property
+    def config_dir(self) -> Path | None:
+        return self._config_dir
+
+    @property
+    def work_dir(self) -> Path:
+        return self._work_dir
+
+    @property
+    def config_path(self) -> Path | None:
+        return self._config_path
+
+    @property
+    def config_sha256(self) -> str | None:
+        return self._config_sha256
+
+    @property
+    def _input_base(self) -> Path:
+        """Anchor for input dirs: the config's directory, or work_dir for in-memory configs."""
+        return self._config_dir if self._config_dir is not None else self._work_dir
+
+    def resolve_input_dir(self, rel: str) -> Path:
+        """Resolve an input dir (prompts/rubrics/pricing) under config_dir; absolute paths pass through."""
+        p = Path(rel).expanduser()
+        return (p if p.is_absolute() else self._input_base / p).resolve()
+
+    @property
+    def study_dir(self) -> Path:
+        """Output study directory, anchored to work_dir (CWD); absolute output_dir passes through."""
+        out = Path(self.output_dir).expanduser()
+        base = out if out.is_absolute() else self._work_dir / out
+        return (base / self.study).resolve()
+
+    def grader_spec(self, name: str) -> GraderSpec:
+        """Resolve a facets.grader entry. Raises ConfigError if unresolvable."""
+        if name in self.graders:
+            return self.graders[name]
+        if "/" in name:  # bare model id used directly as a grader
+            return GraderSpec(model=name)
+        raise ConfigError(f"grader '{name}' is not defined under graders: and is not a model id")
+
+
+def load_config(path: "str | Path", *, work_dir: "str | Path | None" = None) -> ExperimentConfig:
+    """Load and validate an experiment config YAML file.
+
+    Inputs (prompts/rubrics) anchor to the config file's directory; outputs (the
+    study dir) anchor to `work_dir`, defaulting to the current working directory.
+    """
+    p = Path(path).expanduser().resolve()
+    if not p.is_file():
+        raise ConfigError(f"config file not found: {p}")
+    raw = p.read_bytes()
+    try:
+        data = yaml.safe_load(raw)
+    except yaml.YAMLError as e:
+        raise ConfigError(f"invalid YAML in {p}: {e}") from e
+    if not isinstance(data, dict):
+        raise ConfigError(f"config root must be a YAML mapping: {p}")
+    try:
+        cfg = ExperimentConfig.model_validate(data)
+    except ValidationError as e:
+        raise ConfigError(f"invalid config {p}:\n{e}") from e
+    cfg._config_dir = p.parent
+    cfg._work_dir = Path(work_dir).expanduser().resolve() if work_dir is not None else Path.cwd()
+    cfg._config_path = p
+    cfg._config_sha256 = sha256_hex(raw)
+    return cfg
+
+
+def config_to_jsonable(cfg: ExperimentConfig) -> dict[str, Any]:
+    """Config as a JSON-ready dict using YAML key names (for manifests)."""
+    return cfg.model_dump(mode="json", by_alias=True)

itemeval/_errors.py

@@ -0,0 +1,25 @@
+"""Exception hierarchy for itemeval."""
+
+
+class ItemevalError(Exception):
+    """Base class for all itemeval errors."""
+
+
+class ConfigError(ItemevalError):
+    """YAML shape/validation failures and bad config references."""
+
+
+class AdapterError(ItemevalError):
+    """Dataset load or field-mapping failures."""
+
+
+class TemplateError(ItemevalError):
+    """Missing template file or required placeholder."""
+
+
+class StoreError(ItemevalError):
+    """Parquet schema or IO problems."""
+
+
+class BudgetError(ItemevalError):
+    """Pricing refresh or estimator failures."""

itemeval/_item.py

@@ -0,0 +1,29 @@
+"""Canonical benchmark item model."""
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class Item(BaseModel):
+    """Canonical benchmark item: what adapters produce and both stages consume."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    id: str
+    input: str
+    target: str = ""
+    grading_scheme: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    @field_validator("id", mode="before")
+    @classmethod
+    def _coerce_id(cls, v: Any) -> str:
+        return str(v)
+
+    @field_validator("input")
+    @classmethod
+    def _non_empty_input(cls, v: str) -> str:
+        if not v or not v.strip():
+            raise ValueError("Item.input must be non-empty")
+        return v

itemeval/_manifest.py

@@ -0,0 +1,196 @@
+"""Run manifests: full reproducibility record, one JSON per generate/grade run."""
+
+import json
+import platform
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from itemeval._config import config_to_jsonable
+from itemeval._util import atomic_write_bytes, canonical_json, sha256_hex, utc_now_iso
+
+if TYPE_CHECKING:
+    from itemeval._prepare import PreparedStudy
+    from itemeval.store._layout import StudyPaths
+
+MANIFEST_VERSION = 1
+_TRACKED_PACKAGES = ("inspect-ai", "pandas", "pyarrow", "pydantic", "pyyaml", "datasets")
+
+
+class DatasetManifest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    adapter: str
+    split: str
+    name: str | None = None
+    revision_requested: str | None
+    revision_resolved: str
+    n_items: int
+    items_hash: str  # 12 hex over (id, input-hash) pairs in loaded order
+
+
+class TemplateManifest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    name: str  # the reference as written, e.g. "standard" or "builtin:standard"
+    source: str  # "local" | "builtin"
+    path: str  # local path (relative to config_dir where possible), or "builtin:<subdir>/<name>.md"
+    sha256: str
+
+
+class ConditionManifest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    slug: str
+    payload: dict
+
+
+class Manifest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    manifest_version: int = MANIFEST_VERSION
+    run_id: str
+    stage: Literal["generate", "grade"]
+    study: str
+    created_at: str
+    itemeval_version: str
+    python_version: str
+    packages: dict[str, str]
+    config_path: str
+    config_sha256: str
+    config: dict
+    datasets: list[DatasetManifest]
+    solver_templates: list[TemplateManifest]
+    rubric_templates: list[TemplateManifest]
+    models: list[str]
+    graders: dict[str, dict]
+    sampling_requested: dict
+    sampling_effective: dict[str, Any] | None = None  # backfilled post-run, per condition
+    # backfilled post-run, per condition: {provider, base_url, served_model} — the
+    # endpoint/account/version that actually answered (which dashboard billed it).
+    endpoints_effective: dict[str, Any] | None = None
+    seed: int | None
+    policy: str
+    replications_requested: int
+    replications_effective: int
+    items_limit: int | None
+    batch: bool | int | None
+    grid_generate: list[ConditionManifest]
+    grid_grade: list[ConditionManifest]
+    conditions_run: list[str]
+    estimate_usd: float | None
+    cache: bool
+
+
+def _pkg_version(name: str) -> str:
+    try:
+        return version(name)
+    except PackageNotFoundError:
+        return "unknown"
+
+
+def _items_hash(items) -> str:
+    pairs = [[it.id, sha256_hex(it.input.encode("utf-8"))[:12]] for it in items]
+    return sha256_hex(canonical_json(pairs).encode("utf-8"))[:12]
+
+
+def _rel_path(path: str, base: Path) -> str:
+    try:
+        return str(Path(path).relative_to(base))
+    except ValueError:
+        return path
+
+
+def _template_manifest(t, base: Path) -> "TemplateManifest":
+    # built-in templates keep their package-relative id (machine-independent);
+    # local templates are recorded relative to config_dir where possible.
+    path = t.path if t.source == "builtin" else _rel_path(t.path, base)
+    return TemplateManifest(name=t.name, source=t.source, path=path, sha256=t.sha256)
+
+
+def build_manifest(
+    prep: "PreparedStudy",
+    stage: str,
+    run_id: str,
+    conditions_run: "list[str]",
+    estimate_usd: "float | None",
+) -> Manifest:
+    cfg = prep.config
+    base = cfg.config_dir or cfg.work_dir
+    used_graders = {
+        name: cfg.grader_spec(name).model_dump(mode="json") for name in cfg.facets.grader
+    }
+    sampling = cfg.solvers.model_dump(mode="json")
+    sampling.pop("models", None)
+    return Manifest(
+        run_id=run_id,
+        stage=stage,  # type: ignore[arg-type]
+        study=cfg.study,
+        created_at=utc_now_iso(),
+        itemeval_version=_pkg_version("itemeval"),
+        python_version=platform.python_version(),
+        packages={p: _pkg_version(p) for p in _TRACKED_PACKAGES},
+        config_path=str(cfg.config_path) if cfg.config_path else "(in-memory)",
+        config_sha256=cfg.config_sha256 or "",
+        config=config_to_jsonable(cfg),
+        datasets=[
+            DatasetManifest(
+                id=ds.dataset_id,
+                adapter=ds.adapter,
+                split=ds.split,
+                name=ds.name,
+                revision_requested=ds.revision_requested,
+                revision_resolved=ds.revision,
+                n_items=len(ds.items),
+                items_hash=_items_hash(ds.items),
+            )
+            for ds in prep.datasets
+        ],
+        solver_templates=[_template_manifest(t, base) for t in prep.solver_templates.values()],
+        rubric_templates=[_template_manifest(t, base) for t in prep.rubric_templates.values()],
+        models=list(cfg.solvers.models),
+        graders=used_graders,
+        sampling_requested=sampling,
+        seed=cfg.solvers.seed,
+        policy=prep.plan.policy,
+        replications_requested=cfg.facets.replications,
+        replications_effective=prep.plan.replications,
+        items_limit=prep.plan.items_limit,
+        batch=prep.plan.batch,
+        grid_generate=[
+            ConditionManifest(id=c.id, slug=c.slug, payload=c.payload) for c in prep.grid.generate
+        ],
+        grid_grade=[
+            ConditionManifest(id=c.id, slug=c.slug, payload=c.payload) for c in prep.grid.grade
+        ],
+        conditions_run=conditions_run,
+        estimate_usd=estimate_usd,
+        cache=cfg.cache,
+    )
+
+
+def write_manifest(manifest: Manifest, paths: "StudyPaths") -> Path:
+    path = paths.manifests_dir / f"{manifest.run_id}.json"
+    payload = json.dumps(manifest.model_dump(mode="json"), indent=2, ensure_ascii=False)
+    atomic_write_bytes(path, (payload + "\n").encode("utf-8"))
+    return path
+
+
+def finalize_manifest(
+    manifest_path: Path,
+    sampling_effective: "dict[str, Any] | None" = None,
+    endpoints_effective: "dict[str, Any] | None" = None,
+) -> None:
+    """Backfill per-condition effective values after the run completes:
+    sampling params (generate) and/or the resolved endpoint per condition."""
+    data = json.loads(manifest_path.read_text(encoding="utf-8"))
+    if sampling_effective is not None:
+        data["sampling_effective"] = sampling_effective
+    if endpoints_effective is not None:
+        data["endpoints_effective"] = endpoints_effective
+    payload = json.dumps(data, indent=2, ensure_ascii=False)
+    atomic_write_bytes(manifest_path, (payload + "\n").encode("utf-8"))