itemeval 0.1.0__tar.gz

itemeval-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,42 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: lint + test (py${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv + Python ${{ matrix.python-version }}
+        uses: astral-sh/setup-uv@v8.2.0
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Sync (locked, dev group)
+        run: uv sync --locked
+
+      - name: Ruff lint
+        run: uv run ruff check .
+
+      - name: Ruff format check
+        run: uv run ruff format --check .
+
+      # Unit tests only; the `network` test (HF Hub) is excluded to keep CI
+      # hermetic — run it manually per DEVELOPMENT.md when touching the adapter.
+      - name: Pytest
+        run: uv run pytest -m "not network"

itemeval-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,32 @@
+name: Release
+
+# Publishes to PyPI via trusted publishing (OIDC) — no API token stored.
+# Triggered when a GitHub Release is published; the workflow is taken from the
+# default branch, so it fires for tags created before this file existed.
+#
+# One-time PyPI setup (project settings -> Publishing -> add a trusted publisher):
+#   Owner: luozm | Repository: itemeval | Workflow: release.yml | Environment: (blank)
+on:
+  release:
+    types: [published]
+
+jobs:
+  pypi-publish:
+    name: Build + publish to PyPI
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write   # mint the OIDC token PyPI trusted publishing verifies
+      contents: read
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.2.0
+
+      - name: Build sdist + wheel
+        run: uv build
+
+      # --trusted-publishing always: fail loudly if OIDC isn't available rather
+      # than silently falling back to looking for credentials.
+      - name: Publish
+        run: uv publish --trusted-publishing always

itemeval-0.1.0/.github/workflows/sync-wiki.yml

@@ -0,0 +1,33 @@
+name: Sync wiki
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "docs/wiki/**"
+
+permissions:
+  contents: write
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Push docs/wiki to the wiki repo
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git clone "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.wiki.git" wiki
+          rsync -av --delete --exclude .git docs/wiki/ wiki/
+          cd wiki
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add -A
+          if git diff --cached --quiet; then
+            echo "Wiki already up to date."
+            exit 0
+          fi
+          git commit -m "Sync wiki from docs/wiki @ ${GITHUB_SHA::7}"
+          git push

itemeval-0.1.0/.gitignore

@@ -0,0 +1,26 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+
+# Environments & secrets
+.venv/
+.env
+
+# Tooling caches
+.pytest_cache/
+.ruff_cache/
+.coverage
+.coverage.*
+htmlcov/
+
+# Demo study outputs (itemeval generate/grade/export artifacts)
+studies/
+
+# Internal build-time design contract — kept locally, not published
+docs/DESIGN.md
+
+# OS
+.DS_Store

itemeval-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

itemeval-0.1.0/CHANGELOG.md

@@ -0,0 +1,117 @@
+# Changelog
+
+All notable changes to itemeval are documented here. Format follows
+[Keep a Changelog](https://keepachangelog.com); versioning follows
+[SemVer](https://semver.org) (pre-1.0: minor bumps may break APIs).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-06-10
+
+First public release. Item-level LLM evaluation over any inspect_ai-supported
+provider, with a two-stage generate/grade pipeline, long-format item-response
+export, and a budget layer.
+
+### Added
+- Core data model and config (M1): canonical `Item` model; full pydantic
+  experiment-config schema validating the README YAML sketch as-is
+  (`load_config`); content-derived stable condition ids; facet grid expansion
+  with full crossing.
+- HuggingFace benchmark adapter (M1): field-mapping spec → canonical items,
+  revision pinned at first run via a per-study `dataset_locks.json`.
+- Run manifests (M1): dataset revisions, template content hashes, model ids,
+  requested sampling params (effective values backfilled per condition after
+  each run), package versions, full condition grid — one JSON per run.
+- Generate stage (M2): one inspect task per (model × prompt × model-config)
+  cell, `epochs` = replications, thinking/reasoning toggles as model-config
+  facets, requested vs effective sampling params recorded per row, resumable
+  solutions parquet store + raw `.eval` log index.
+- Grade stage (M3): verifiable scorers (exact match / multiple choice /
+  numeric, $0) and judge-as-task (grading dataset built from stored solutions,
+  judge temperature pinned to 0, prompt caching enabled); strict structured
+  score parsing with parse failures flagged in-table, never dropped;
+  re-runnable per (grader × rubric) without touching solutions.
+- Export (M4): long-format gradings table (45 columns: scores, judge
+  reasoning, tokens, USD, latency, full provenance), parquet + CSV mirrors,
+  per-run cost ledger attributed generation vs grading with internal
+  reconciliation check.
+- Budget layer (M5): packaged pricing seed + OpenRouter pricing refresh,
+  per-stage dry-run estimator, `confirm_above_usd` gate (exit 3) and
+  non-overridable `max_usd` cap (exit 4), `dev`/`full-interactive`/
+  `full-batch` policies, batch-API wiring with documented ~50% discount
+  approximation.
+- CLI (M6): `estimate | generate | grade | export | status` with consistent
+  UX, `--json` output, repeatable `--condition/--grader/--rubric` filters,
+  resumability and grid-completion reporting.
+- `mockllm/*` pass-through: any mock model id runs the full pipeline free and
+  deterministically (used by all demos and tests; `configs/usamo_demo.yaml`).
+- Public Python API: the pipeline is drivable programmatically as well as via
+  the CLI — `prepare_study`, `estimate_study`, `run_generate`, `run_grade`,
+  `export_study`, `build_status` exported from `itemeval` (lazily, so
+  `import itemeval` stays light). The budget confirmation gate remains a
+  CLI-layer feature.
+- Dependency: `datasets` (HuggingFace) for the HF adapter.
+- Built-in template library: prompts `minimal`/`standard` and rubric `standard`
+  ship inside the package and are referenced as `builtin:<name>`. A bare name
+  still resolves to a local file under `prompts_dir`/`rubrics_dir`; the two
+  namespaces are distinct and never silently shadow each other — each template
+  is recorded in the run manifest with its `source` (`local`/`builtin`) and
+  content hash, and built-in templates record a machine-independent path.
+- `itemeval init DIR [--with-templates] [--force]`: scaffold a runnable starter
+  study (`config.yaml`). `--with-templates` also copies the referenced built-in
+  prompts/rubrics locally as editable starters. Makes `pip install itemeval`
+  usable without cloning the repo.
+- `solvers.on_empty` policy (`skip` default / `rerun` / `grade`) for completed
+  generations that produced no gradable text (empty/blank `solution`, no API
+  error — e.g. a reasoning model whose token budget was spent entirely on
+  hidden reasoning). Empty no-error completions are a distinct channel from API
+  errors (re-attempted) and parse failures (final): `skip` excludes them from
+  grading, `rerun` also makes them eligible for regeneration on the next
+  `generate`, `grade` sends them to the judge as-is. They are always surfaced —
+  `grade` reports the count and stop-reason breakdown, and `status` gains an
+  `empty` column — never silently folded into a green "complete".
+- Provider/endpoint provenance for cost attribution: `ledger.parquet` gains a
+  `provider` column (the inspect prefix of `model`), and run manifests gain
+  `endpoints_effective` per condition (`{provider, base_url, served_model}`,
+  backfilled after the run) — recording which provider, endpoint, and
+  provider-returned model snapshot actually answered. `base_url` is null on the
+  provider's default endpoint; a non-null value flags traffic routed elsewhere
+  (Azure/proxy/gateway).
+
+### Changed
+- **Path resolution split by intent** (behavior change). Inputs (`prompts_dir`,
+  `rubrics_dir`, `budget.pricing_path`) still anchor to the config file's
+  directory; outputs (`output_dir`, i.e. the study tree) now anchor to a **work
+  directory** defaulting to the current directory, never the config dir or the
+  installed package. New `-C/--base-dir` (CLI) and `load_config(work_dir=...)`
+  (Python) override the output anchor. The example configs drop their `../`
+  prefixes accordingly.
+- Default `facets.prompt` / `facets.rubric` are now `[builtin:standard]`
+  (were `[default]`, which referenced a template that never existed).
+- Template references and validation moved ahead of study-directory creation:
+  an unresolved template now fails before any output directory is written.
+
+### Packaging
+- Provider-SDK optional extras (`openai`, `anthropic`, `google`, `all`),
+  mirroring inspect_ai's lazy provider imports. Install the extra for the
+  provider you run, e.g. `pip install itemeval[openai]` — the `openai` extra
+  also covers OpenRouter and other OpenAI-compatible providers. The base
+  install stays SDK-free; running a real provider without its extra raises
+  inspect_ai's `PrerequisiteError` with the install hint.
+- Ship a `py.typed` marker (PEP 561): downstream type checkers now see
+  itemeval's annotations. Added the `Typing :: Typed` and Python 3.11/3.12
+  classifiers.
+- Relaxed the `pyarrow` (`>=24` → `>=15`) and `datasets` (`>=5` → `>=3`)
+  lower bounds to the oldest versions whose APIs we actually use, easing
+  co-installation; dev/CI still pin the latest via `uv.lock`. The full test
+  suite passes at both the floor and the locked versions.
+- Expanded `[project.urls]` (Homepage, Documentation → wiki, Changelog, Issues)
+  and switched the README's PyPI-facing links to absolute GitHub URLs.
+- Minimum Python is now 3.11 (was 3.10). The tested dependency stack resolves
+  pandas 3.x, which requires Python >=3.11, so 3.10 could only ever install a
+  different (pandas 2.x) stack that was never tested. Floor now matches the
+  tested stack; `uv.lock` reconciled to a single resolution (dropped the
+  3.10-only `exceptiongroup`/`tomli`/`async-timeout`/`pytz` backports).
+
+[Unreleased]: https://github.com/luozm/itemeval/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/luozm/itemeval/releases/tag/v0.1.0

itemeval-0.1.0/CLAUDE.md

@@ -0,0 +1,35 @@
+# CLAUDE.md — itemeval (package development)
+
+Publishable Python package: item-level LLM evaluation on inspect_ai.
+`README.md` is the spec; `ROADMAP.md` is the milestone plan — keep it updated
+as milestones complete. `DEVELOPMENT.md` defines the inspect_ai upgrade
+pipeline and the versioning/release process — follow it for any dependency
+bump or release; update `CHANGELOG.md` ([Unreleased]) in the same change that
+makes a user-visible difference. Any study consuming this package lives in its
+own separate repo; never put study-specific content (a particular study's
+datasets, rubric texts, or analysis) in this package.
+
+## Python environment
+
+- Managed by `uv`. All Python runs against `./.venv`; system Python is never used.
+- Invoke by absolute path, never activate: `./.venv/bin/python -m pytest`.
+  Never emit `source .venv/bin/activate`.
+- New deps: `uv add <pkg>` (runtime) / `uv add --dev <pkg>` (dev). Never call
+  pip directly. `pyproject.toml` carries ranges; `uv.lock` pins exactly and is
+  committed.
+- Recreate from scratch: `uv sync`. `./.venv` is disposable.
+- Supports Python >=3.11 (don't use 3.12+ syntax); develop on 3.12. Floor is
+  3.11 because the tested stack pulls pandas 3.x, which requires >=3.11.
+
+## Conventions
+
+- src layout: code in `src/itemeval/`; tests import the installed package.
+- Public API is exported from `itemeval/__init__.py`; everything else is
+  internal and free to refactor. Prefix private modules with `_`.
+- pydantic models for all config/data schemas; YAML configs validated at load.
+- Lint/format: `./.venv/bin/python -m ruff check . && ./.venv/bin/python -m ruff format .`
+- Tests: `./.venv/bin/python -m pytest`. Unit tests must not call paid APIs;
+  anything touching providers is mocked or marked for manual runs.
+- No real API keys in tests, fixtures, or examples.
+- Conventional commits (feat:/fix:/docs:/test:/refactor:); update CHANGELOG.md
+  for user-visible changes once releases start.

itemeval-0.1.0/DEVELOPMENT.md

@@ -0,0 +1,92 @@
+# Development Guide
+
+Process documentation for maintaining itemeval. For coding conventions see
+`CLAUDE.md`; for milestones see `ROADMAP.md`.
+
+## Dependency policy
+
+- `pyproject.toml` declares **ranges** (lower bounds, e.g. `inspect-ai>=0.3.239`)
+  — this is the contract published to package users.
+- `uv.lock` pins **exact versions** of everything — this is what `uv sync`
+  reproduces locally and in CI. Always committed.
+- Add/remove only via `uv add <pkg>` / `uv remove <pkg>`; never edit dependency
+  lists by hand and never call pip.
+- Upper-bound pins (`<X`) are allowed only as a temporary response to a known
+  breakage, with a linked issue and removal plan.
+
+## inspect_ai upgrade pipeline
+
+inspect-ai is the load-bearing dependency and releases frequently. Upgrades are
+**deliberate, never incidental** — routine `uv sync` keeps using the lockfile
+pin, so versions only move when we move them.
+
+Cadence: at the start of each ROADMAP milestone, and before any large paid run
+in a consuming study.
+
+1. **Check what's new**
+   ```bash
+   uv tree --package inspect-ai --depth 0     # current pinned version
+   ```
+   Read the release notes: https://github.com/UKGovernmentBEIS/inspect_ai/releases
+   Watch for: model-provider changes, batch/caching behavior, log-format (.eval
+   schema) changes, dataframe API (`samples_df`/`evals_df`) changes.
+2. **Upgrade on a branch**
+   ```bash
+   git checkout -b chore/bump-inspect-ai
+   uv lock --upgrade-package inspect-ai && uv sync
+   ```
+3. **Unit tests** (no API calls): `./.venv/bin/python -m pytest`
+4. **Live smoke test** (manual, costs cents): run the consuming study's pilot
+   config at `dev` scope end-to-end (generate → grade → export) and confirm
+   the export schema and cost ledger are unchanged.
+5. **Commit** the lockfile bump: `chore: bump inspect-ai 0.3.X -> 0.3.Y`, with
+   any behavior notes in the body. Merge.
+6. **On breakage**: pin a temporary upper bound in `pyproject.toml`
+   (`inspect-ai>=0.3.X,<0.3.Y`), open an issue describing the incompatibility,
+   and remove the bound when fixed.
+
+Once CI exists (ROADMAP M7): add a scheduled weekly GitHub Actions job (or
+Renovate/Dependabot) that opens the upgrade PR automatically; steps 3–5 run in
+CI, step 4 stays manual.
+
+All other dependencies: `uv lock --upgrade && uv sync` quarterly, same
+branch-test-commit flow, less scrutiny.
+
+## Versioning discipline
+
+**Semantic versioning**, version lives in one place: `pyproject.toml`
+(`itemeval.__version__` reads it via package metadata — never duplicate it).
+
+- **Pre-1.0 (now)**: `0.MINOR.PATCH`. Minor bumps may break APIs (expected at
+  this stage and noted in the changelog); patch bumps are fixes only.
+  Between releases the version carries a `.devN` suffix (e.g. `0.1.0.dev0`).
+- **Post-1.0**: MAJOR = breaking, MINOR = backwards-compatible features,
+  PATCH = backwards-compatible fixes. Breaking changes are deprecated with a
+  runtime warning for at least one minor release before removal.
+
+**CHANGELOG.md** follows [Keep a Changelog](https://keepachangelog.com):
+user-visible changes are added to the `[Unreleased]` section in the same PR
+that makes them — never reconstructed at release time.
+
+PyPI publishing uses **trusted publishing** (OIDC from GitHub Actions — no API
+token stored). One-time setup: on PyPI, add a trusted publisher for the project
+(owner `luozm`, repo `itemeval`, workflow `release.yml`, environment blank). The
+publish itself runs in `.github/workflows/release.yml`, triggered when a GitHub
+release is published; locally you only build/tag.
+
+**Release checklist** (applies from v0.1.0, ROADMAP M7):
+
+1. Tests and lint green: `./.venv/bin/python -m pytest && ./.venv/bin/python -m ruff check .`
+2. Move `[Unreleased]` entries under a new `[X.Y.Z] - YYYY-MM-DD` heading.
+3. Set `version = "X.Y.Z"` in `pyproject.toml` (drop the `.devN`).
+4. Optionally verify the build locally: `uv build` (the same command CI runs).
+5. Commit `release: vX.Y.Z`; tag and push: `git tag vX.Y.Z && git push origin main --tags`.
+6. Create a GitHub release from the tag (body = the changelog section). Publishing
+   to PyPI is automatic: the `release: published` event triggers `release.yml`,
+   which runs `uv build && uv publish` via trusted publishing. Watch the Actions
+   run and confirm the new version appears on PyPI.
+7. Bump to the next dev version (e.g. `0.2.0.dev0`) in a follow-up commit.
+
+Consuming studies pin itemeval like any dependency: editable path source during
+development, exact-version pin from PyPI once published — their `uv.lock` plus
+run manifests (which record `itemeval.__version__`) keep results reproducible.

itemeval-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zhimeng (Brian) Luo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.