evalkit-harness 0.1.0__tar.gz

evalkit_harness-0.1.0/.gitignore

@@ -0,0 +1,26 @@
+.venv/
+__pycache__/
+*.pyc
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.harness/
+.evalkit/
+.env
+.DS_Store
+
+# personal notes, not for the public repo
+Portfolio notes.md
+.venv-py*/
+runs/
+heldout/*
+!heldout/MANIFEST.json
+
+# Streamlit local secrets
+.streamlit/secrets.toml
+
+# Claude Code local settings
+.claude/settings.local.json
+.claude/scheduled_tasks.lock

evalkit_harness-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Biplob Chakraborty
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalkit_harness-0.1.0/PKG-INFO

@@ -0,0 +1,70 @@
+Metadata-Version: 2.5
+Name: evalkit-harness
+Version: 0.1.0
+Summary: Evaluate your own task across Claude and OpenAI models: grounded, calibrated, cost-aware, with confidence intervals.
+Project-URL: Homepage, https://github.com/biplob819/evalkit
+Project-URL: Issues, https://github.com/biplob819/evalkit/issues
+Author: Biplob Chakraborty
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,benchmark,evaluation,groundedness,llm,llm-as-judge
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pyyaml>=6
+Provides-Extra: agents
+Requires-Dist: duckdb>=1.1; extra == 'agents'
+Requires-Dist: pytz>=2026.6; extra == 'agents'
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=1; extra == 'anthropic'
+Provides-Extra: demo
+Requires-Dist: psycopg[binary]>=3.2; extra == 'demo'
+Requires-Dist: streamlit>=1.40; extra == 'demo'
+Provides-Extra: openai
+Requires-Dist: openai>=2; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# EvalKit
+
+<!-- evalkit:headline:start -->
+<p align="center"><img src="https://raw.githubusercontent.com/biplob819/evalkit/main/docs/assets/headline.svg" alt="On 200 PubMedQA questions, 4 models (claude-sonnet-5-5, claude-haiku-5-5, gpt-6.1-sol, gpt-6-luna) answered from the abstract and cited its sentences; two judges from different families (claude-opus-5-5 and gpt-6.1-sol) checked every claim. The most grounded model was gpt-6.1-sol, correct and fully grounded on 57.5% (95% CI 50.5%–64.0%) by claude-opus-5-5, 46.0% (95% CI 39.0%–53.0%) by gpt-6.1-sol; it is separated from every other model after a Holm correction (gpt-6.1-sol also judged its own answers; claude-opus-5-5 agrees); the most accurate model was claude-sonnet-5-5 (77.0% correct labels); the headline also requires every claim to be supported by the sentences the answer cites. Caveats: judge not calibrated for this task (see docs/calibrate-your-task.md). Treat these numbers as provisional." width="760"></p>
+
+On 200 PubMedQA questions, 4 models (claude-sonnet-5-5, claude-haiku-5-5, gpt-6.1-sol, gpt-6-luna) answered from the abstract and cited its sentences; two judges from different families (claude-opus-5-5 and gpt-6.1-sol) checked every claim. The most grounded model was gpt-6.1-sol, correct and fully grounded on 57.5% (95% CI 50.5%–64.0%) by claude-opus-5-5, 46.0% (95% CI 39.0%–53.0%) by gpt-6.1-sol; it is separated from every other model after a Holm correction (gpt-6.1-sol also judged its own answers; claude-opus-5-5 agrees); the most accurate model was claude-sonnet-5-5 (77.0% correct labels); the headline also requires every claim to be supported by the sentences the answer cites. Caveats: judge not calibrated for this task (see docs/calibrate-your-task.md). Treat these numbers as provisional.
+
+[Full report](https://biplob819.github.io/evalkit/) · results in [`docs/results/`](https://github.com/biplob819/evalkit/tree/main/docs/results)
+
+```bash
+pip install 'evalkit-harness[anthropic,openai]'
+evalkit init   # scaffold a project and a three-case sample task
+evalkit run evalkit.yaml --task tasks/my_task.yaml --n 3 --dry-run   # estimate
+evalkit run evalkit.yaml --task tasks/my_task.yaml --n 3   # needs keys in .env
+```
+
+Questions from [PubMedQA](https://pubmedqa.github.io/) (Jin et al., 2019, MIT licence); see [DATA_CARD.md](https://github.com/biplob819/evalkit/blob/main/DATA_CARD.md). Research benchmark, not medical advice.
+<!-- evalkit:headline:end -->
+
+Evaluate **your own task** across Claude and OpenAI models: accuracy, groundedness, citation validity, cost and latency side by side, with confidence intervals and two cross-family judges whose error rate is measured against held-out hand labels.
+
+> **Status: v0.1.0, the first public release.** To work on EvalKit itself, clone it and run `uv sync --all-extras`.
+
+**Start here:** [Quickstart](https://github.com/biplob819/evalkit/blob/main/docs/quickstart.md) (a first result in under five minutes) · [Write your own task](https://github.com/biplob819/evalkit/blob/main/docs/writing-a-task.md) (JSONL or CSV) · [Control cost](https://github.com/biplob819/evalkit/blob/main/docs/cost.md) · [Calibrate the judges](https://github.com/biplob819/evalkit/blob/main/docs/calibrate-your-task.md) · [Scorers and adapters](https://github.com/biplob819/evalkit/blob/main/docs/plugins.md) · [Run it in CI](https://github.com/biplob819/evalkit/blob/main/examples/github-action/evalkit.yml) · [Versioning](https://github.com/biplob819/evalkit/blob/main/docs/versioning.md)
+
+Built-in tasks: `pubmedqa` (yes / no / maybe research questions) and `scifact` (supports / refutes / not enough info for scientific claims); see [DATA_CARD.md](https://github.com/biplob819/evalkit/blob/main/DATA_CARD.md).
+
+No telemetry: data goes only to the providers you configure ([PRIVACY.md](https://github.com/biplob819/evalkit/blob/main/PRIVACY.md)). Security and key handling: [SECURITY.md](https://github.com/biplob819/evalkit/blob/main/SECURITY.md). Contributions welcome: [CONTRIBUTING.md](https://github.com/biplob819/evalkit/blob/main/CONTRIBUTING.md).
+
+The package is published as `evalkit-harness` (the name `evalkit` on PyPI belongs to an unrelated project). The command and the import are both `evalkit`.
+
+- Product: [EvalKit PRD](https://github.com/biplob819/evalkit/blob/main/EvalKit%20PRD.md)
+- Design: [EvalKit Tech Spec](https://github.com/biplob819/evalkit/blob/main/EvalKit%20Tech%20Spec.md)
+- Build status: `uv run python scripts/status.py`
+
+MIT licence.

evalkit_harness-0.1.0/README.md

@@ -0,0 +1,36 @@
+# EvalKit
+
+<!-- evalkit:headline:start -->
+<p align="center"><img src="https://raw.githubusercontent.com/biplob819/evalkit/main/docs/assets/headline.svg" alt="On 200 PubMedQA questions, 4 models (claude-sonnet-5-5, claude-haiku-5-5, gpt-6.1-sol, gpt-6-luna) answered from the abstract and cited its sentences; two judges from different families (claude-opus-5-5 and gpt-6.1-sol) checked every claim. The most grounded model was gpt-6.1-sol, correct and fully grounded on 57.5% (95% CI 50.5%–64.0%) by claude-opus-5-5, 46.0% (95% CI 39.0%–53.0%) by gpt-6.1-sol; it is separated from every other model after a Holm correction (gpt-6.1-sol also judged its own answers; claude-opus-5-5 agrees); the most accurate model was claude-sonnet-5-5 (77.0% correct labels); the headline also requires every claim to be supported by the sentences the answer cites. Caveats: judge not calibrated for this task (see docs/calibrate-your-task.md). Treat these numbers as provisional." width="760"></p>
+
+On 200 PubMedQA questions, 4 models (claude-sonnet-5-5, claude-haiku-5-5, gpt-6.1-sol, gpt-6-luna) answered from the abstract and cited its sentences; two judges from different families (claude-opus-5-5 and gpt-6.1-sol) checked every claim. The most grounded model was gpt-6.1-sol, correct and fully grounded on 57.5% (95% CI 50.5%–64.0%) by claude-opus-5-5, 46.0% (95% CI 39.0%–53.0%) by gpt-6.1-sol; it is separated from every other model after a Holm correction (gpt-6.1-sol also judged its own answers; claude-opus-5-5 agrees); the most accurate model was claude-sonnet-5-5 (77.0% correct labels); the headline also requires every claim to be supported by the sentences the answer cites. Caveats: judge not calibrated for this task (see docs/calibrate-your-task.md). Treat these numbers as provisional.
+
+[Full report](https://biplob819.github.io/evalkit/) · results in [`docs/results/`](https://github.com/biplob819/evalkit/tree/main/docs/results)
+
+```bash
+pip install 'evalkit-harness[anthropic,openai]'
+evalkit init   # scaffold a project and a three-case sample task
+evalkit run evalkit.yaml --task tasks/my_task.yaml --n 3 --dry-run   # estimate
+evalkit run evalkit.yaml --task tasks/my_task.yaml --n 3   # needs keys in .env
+```
+
+Questions from [PubMedQA](https://pubmedqa.github.io/) (Jin et al., 2019, MIT licence); see [DATA_CARD.md](https://github.com/biplob819/evalkit/blob/main/DATA_CARD.md). Research benchmark, not medical advice.
+<!-- evalkit:headline:end -->
+
+Evaluate **your own task** across Claude and OpenAI models: accuracy, groundedness, citation validity, cost and latency side by side, with confidence intervals and two cross-family judges whose error rate is measured against held-out hand labels.
+
+> **Status: v0.1.0, the first public release.** To work on EvalKit itself, clone it and run `uv sync --all-extras`.
+
+**Start here:** [Quickstart](https://github.com/biplob819/evalkit/blob/main/docs/quickstart.md) (a first result in under five minutes) · [Write your own task](https://github.com/biplob819/evalkit/blob/main/docs/writing-a-task.md) (JSONL or CSV) · [Control cost](https://github.com/biplob819/evalkit/blob/main/docs/cost.md) · [Calibrate the judges](https://github.com/biplob819/evalkit/blob/main/docs/calibrate-your-task.md) · [Scorers and adapters](https://github.com/biplob819/evalkit/blob/main/docs/plugins.md) · [Run it in CI](https://github.com/biplob819/evalkit/blob/main/examples/github-action/evalkit.yml) · [Versioning](https://github.com/biplob819/evalkit/blob/main/docs/versioning.md)
+
+Built-in tasks: `pubmedqa` (yes / no / maybe research questions) and `scifact` (supports / refutes / not enough info for scientific claims); see [DATA_CARD.md](https://github.com/biplob819/evalkit/blob/main/DATA_CARD.md).
+
+No telemetry: data goes only to the providers you configure ([PRIVACY.md](https://github.com/biplob819/evalkit/blob/main/PRIVACY.md)). Security and key handling: [SECURITY.md](https://github.com/biplob819/evalkit/blob/main/SECURITY.md). Contributions welcome: [CONTRIBUTING.md](https://github.com/biplob819/evalkit/blob/main/CONTRIBUTING.md).
+
+The package is published as `evalkit-harness` (the name `evalkit` on PyPI belongs to an unrelated project). The command and the import are both `evalkit`.
+
+- Product: [EvalKit PRD](https://github.com/biplob819/evalkit/blob/main/EvalKit%20PRD.md)
+- Design: [EvalKit Tech Spec](https://github.com/biplob819/evalkit/blob/main/EvalKit%20Tech%20Spec.md)
+- Build status: `uv run python scripts/status.py`
+
+MIT licence.

evalkit_harness-0.1.0/changelog.md

@@ -0,0 +1,74 @@
+# Changelog
+
+Format: `[TYPE] [NODE] — summary`. Types: FEAT, FIX, DOCS, HARNESS, RELEASE.
+
+## 2026-11-15
+- [RELEASE] [B2.5] — v0.1.0 prepared: SciFact as a second built-in task, CSV datasets, quickstart / task / plugin / cost / versioning docs, community files, privacy statement, CI action example, public API contract test, post-release smoke job on three OSes
+- [FEAT] [D3.2] — demo page (Streamlit): replay by default with six prepared samples (`evalkit demo-cache`, $0.99 in three runs), live mode on the owner's keys behind the passcode or on a visitor's own keys (D-58), cards that fill in per model, aggregate-only logging; failed or unchecked cards are re-answered by the next cache run
+- [FEAT] [D3.1] — see commit 20c2fbc
+- [FEAT] [B2.3] — README headline chart and sentences generated from the published 200-case run (re-judged, D-52); report deployed to GitHub Pages by workflow (D-47, D-57)
+- [FIX] [C3.4] — first full agent run (300 trials, $3.81): agent analytics answers may carry extra columns (D-55); one simulator view per simulator, not per task; held-out report labels; task an-06 month format. Result: Sonnet 81% vs Sol 82–83% public success, 75% each held-out, no separated gap; agents rarely asked clarifying questions (6 and 16 of 60 ambiguous trials)
+- [FEAT] [D3.1] — demo backend: `compare()` answer cards (live single-judge path with per-model stage timeouts, replay), PubMed fetch by PMID, in-memory store for visitor data, guard (passcode, session rate limit, per-request cap) and a Postgres daily spend counter with atomic reservation (D-53, D-54)
+- [FIX] [C3.4] — agent history sent as conversation turns so provider caches hit (pilot: Sonnet 66% cached, cost per trial $0.040 → $0.018); estimate uses measured pilot tokens (full run $6.16, was $31.52 on placeholders)
+- [FIX] [A2.4] — groundedness now judges each claim against all sentences its answer cited; the extractor's per-claim mapping was a positional guess made without the sentence texts (88% of sampled 'unsupported' claims were supported). The first 200-case headline was withdrawn and the run re-judged (D-52, L-37)
+
+## 2026-11-14
+- [FEAT] [C3.4] — agent scorers (sql_match / attack outcome, violations, steps over minimum, recovery, cost per success), two-level bootstrap with per-simulator views, Holm-adjusted separation matrix, held-out gap, `evalkit agent-run` with estimate, total cap, resume and an Agents report; agent prompts made cache-friendly (Anthropic automatic caching) (D-51)
+- [FEAT] [C3.3] — simulated user on Haiku 5.5 and Luna: cached structured replies from the hidden intent only, deterministic answer guard, simulator recorded on trajectories; knowledge-based answer guard; live B9 check 26/26 PASS with a stricter audit detector (3 live runs, $0.0040) (D-50)
+- [FEAT] [C3.2] — agent sandbox and loop: per-trial read-only DuckDB copy with reset hash check (void trials), locked-down SQL, tool allow-list with argument schemas, one structured-output loop for Claude and GPT agents with step budget, cached turns for free replay (D-49)
+- [FEAT] [C3.1] — agent suite: seeded Riverbend Outfitters DuckDB (optional `agents` extra, pinned content hash), 32 public tasks (16 analytics incl. 8 ambiguous with hidden intent, 16 prompt-injection attacks whose canaries sit in `system_settings`, so quoting an injection is safe and only obeying it leaks), 8 held-out tasks kept off the repo with a committed manifest, loader that re-checks every gold answer against its SQL (D-48)
+- [FIX] [W0.1] — Windows CI: explicit UTF-8 on every text read/write (enforced by an AST test), UTF-8 console output, platform paths in tests (L-32)
+- [FIX] [S2.4] — judge every claim against all evidence (not the extractor's citations); pending claims reported separately; live check re-run: PASS
+- [FEAT] [S2.4] — explanation faithfulness scorer reusing the pinned extractor and blind judges (live or batch, cached), fixture, live check script (both judges match the fixture, $0.027)
+- [FIX] [B2.4] — never overwrite user files (example output, task templates); doctor and run check SDKs and the keys the config needs; README installs the provider extras
+- [FEAT] [B2.4] — onboarding: `evalkit init` (config, .env.example, task template, .gitignore; never overwrites), `evalkit doctor` (Python, SDKs, key names, network, prices age, cache), `evalkit report --example` (bundled example run, no keys or network), caveats in every run/report output; shared outputs written 0644
+- [FEAT] [B2.3] — README headline chart and generated sentences (`evalkit.report.readme`), `scripts/publish_results.py`, GitHub Pages workflow (OWNER D-47)
+- [FIX] [A2.4] — batch holds use 1.5x measured expected output, not the 4,096-token cap (a 200-case run could never start under $20); judges run concurrently (D-46)
+- [FIX] [N1.2] — cost model from the 12-case real run: judge calls per claim (4.5 per answer), measured token assumptions; 200-case estimate $14.13 (D-46)
+- [FIX] [B2.2] — run safety (D-45): total cost cap across resumes, judge-set lock per run folder, no live re-pay of open batches, dry run without keys, exit 2 with masked messages for every unexpected error
+- [FEAT] [B2.2] — CLI: `evalkit run` (resumable run folders, dry run, cost cap exit 3, judging-pending exit 4, `--live-judges` sharing the batch cache, `--json`), `evalkit calibrate export|label|import|score`, `evalkit report [--publish]`; validation before any call; user YAML tasks with local JSONL datasets
+- [FIX] [A2.5] — a pending run whose records carry no per-judge scores now aggregates (found by the CLI tests)
+- [FEAT] [B2.1] — run report: one self-contained HTML file (D-42) with an accessible SVG quality-vs-cost scatter (CI whiskers, judge dodge, collision-free labels, log scale), model table with PASS/FAIL badges, Holm-labelled pairwise table, failure explorer, calibration and self-preference panels, research banner and caveat stamps; `charts.vl.json` alongside; golden report
+- [FIX] [A2.5] — float totals use `math.fsum` so `summary.json` is identical on Python 3.11 and 3.12 (CI failure)
+- [HARNESS] [W0.1] — golden gate also runs on the minimum supported Python (L-27)
+- [FEAT] [A2.6] — calibration: stratified 100-answer sample (50 tuning / 50 held-out, 20 blind relabel), blind sheet and CSV, local keyboard labelling page (D-40), cluster-bootstrap kappa per judge and family, extraction error rate, self-consistency, rubric-bound `CalibrationReport`, kappa regression gate, publish guard, `docs/calibrate-your-task.md`
+- [FEAT] [A2.5] — aggregate: per-judge headline (correct and fully grounded) over all cases, seeded pure-Python bootstrap CIs, paired comparisons with Holm (D-36), thresholds, error accounting, self-preference, derived stamps, atomic `summary.json`; golden end-to-end run
+- [FEAT] [A2.4] — dual-judge batch: blind rubric with `judge_prompt_hash`, per-claim batch bookkeeping (resume never pays twice, expired items resubmitted), budget holds across resume, verdict/evidence/quote consistency, one live retry, cross-family dual mode
+- [FIX] [A2.4] — four verifier passes; judge blinding per D-38 (no clinical false positives on PQA-L), collect bound covers downloads, per-judge lock
+- [FEAT] [A2.4] — `scripts/planted_check.py`: live B6 check with positive controls; both judges pass
+- [DOCS] [D-34] — judges and extractor at medium effort, NFR2 $20 per 200-case run; D-35..D-38 recorded
+
+- [HARNESS] [W0.1] — Repo skeleton (`evalkit-harness`, src layout, Python ≥3.11), CI and release workflows, build graph (26 nodes) and capability catalog (144 rows), `verify_graph_eligibility.py`, `node_start.py`, `node_converge.py` (12 gates), `secret_scan.py`, `status.py`, every gate with a self-test, decision log seeded D-1 to D-14.
+- [FIX] [W0.1] — Verifier findings: AST-checked proofs under tests/, git-diff errors fail gate 11, check-only node-start writes nothing, self-tests require the defect's identity, wider secret patterns, strict graph and catalog parsers.
+- [HARNESS] [W0.1] — Green. HAR-08 (PyPI name) moved to B2.5 (OWNER D-20).
+- [FEAT] [N1.1] — `evalkit.schemas` (results format v1: benchmark, summary, calibration, config, agent), `evalkit.config` YAML loaders with file-and-field errors, v1 fixtures and round-trip contract tests.
+- [FIX] [N1.1] — Verifier findings: finite floats, required well-formed schema_version, honest RunSummary stamps, deep immutability, strict ints, linked claims and verdicts, loader errors that name the file, duplicate-key YAML rejected.
+- [FEAT] [N1.1] — Green.
+- [FEAT] [N1.3] — `evalkit.runtime`: typed Node contract, canonical-JSON cache keys, content-addressed store with lineage and batch-id meta, async executor (per-provider concurrency, fan-out in order, resume from cache, spend cap counting in-flight estimates), single-writer JSONL, code-version manifest check.
+- [FEAT] [N1.4] — PubMedQA PQA-L loader (pinned checksum, 1,000 rows, cached, `long_answer` dropped), rule-based sentence splitter checked on all 1,000 abstracts, seeded label-stratified sampler, built-in `pubmedqa` task and prompt rendering, `DATA_CARD.md` with the contamination probe spec.
+- [FIX] [N1.3] — Verifier findings: money values must be finite and non-negative, budget reserved inside the concurrency slot, identical in-flight work coalesced, batch ids reused via `submit_once`, corrupt cache entries recomputed, version manifest keyed by node name and hashing MRO plus schemas, writer refuses NaN and late writes.
+- [FEAT] [N1.3] — Green.
+- [FIX] [N1.4] — Verifier findings: splitter recovers 41 wrong merges (initials, units, unclosed brackets, spaced decimals), download verified before an atomic write, network failures raise DatasetError, duplicate ids rejected, data card claims made true or dated, probe spec fully pinned, PubMedQA credited in the README.
+- [FEAT] [N1.4] — Green.
+- [FEAT] [N1.2] — Adapters: Anthropic (Messages, Message Batches) and OpenAI (Chat Completions, Batch) plus OpenAI-compatible servers, behind one interface with entry-point plugins; no sampling params, explicit effort, structured JSON; `prices.yaml` (Claude 5.5 family, gpt-6.1-sol, gpt-6-luna, gpt-6.1-terra), per-call cost, pre-run estimate and reconciliation; retry policy (6 attempts, full jitter, transport errors only); token-bucket pacing.
+- [FIX] [N1.2] — Verifier findings: expired/cancelled batches keep billed results, real SDK error types, damaged batch lines tolerated, duplicate custom_ids refused, explicit effort enforced, token bucket wired into every adapter call, plugin failures isolated, compatible-server URL and key checks, long-context price tiers, judges at low effort (OWNER D-28, estimate $10.35).
+- [FEAT] [N1.2] — Green. Wave 1 (N1.1-N1.4) complete.
+- [FEAT] [A2.2] — `evalkit.scorers`: Scorer protocol, registry and isolated entry-point plugins; label_correct, citation_valid, cost and latency (errors count as wrong).
+- [FIX] [A2.2] — Verifier findings: built-in scorers can never be replaced by plugins, duplicate and mismatched plugin names warn, failed plugins load once, Unicode-normalised labels, strict finite scores, picklable scorers.
+- [FEAT] [A2.1] — Generate and Validate nodes: provider routing, per-provider pools, structured answers with a strict parser, error taxonomy (transport not cached, refusal, schema never retried), `Executor.map_settled`, per-model effort in RunConfig, reference `configs/pubmedqa.yaml`, smoke-run module and script.
+- [FEAT] [A2.2] — Green.
+- [FEAT] [S2.1] — `sql_match`: result-set comparison (column reordering by name, multiset rows, ordered option, numeric tolerance, strict booleans/None/strings) with 21 golden cases.
+- [FIX] [A2.1] — Verifier findings: budget exhaustion and configuration errors stop the run (only transient provider errors are 'transport'), answers must be one JSON object (validate v2), .env parsing, smoke exit codes and labels, tighter OpenAI routing.
+- [FEAT] [S2.2] — Retrieval metrics: recall@k, MRR, nDCG@k (binary or graded), positions as returned, duplicates earn no credit.
+- [FEAT] [A2.1] — Green (real smoke run pending API keys).
+- [FEAT] [A2.3] — Claim extractor: pinned prompt with hashed extractor_version, ExtractClaims node at low effort with structured output, B11 checks (no invented citations, no duplicates, no empty extraction), coverage v1 (D-31).
+- [FIX] [S2.1] — Verifier blocker fixed: unordered matching honours the tolerance exactly (exact-first, grouped, Hopcroft-Karp matching, windowed candidates), exact big-integer comparison, Decimal as numbers, 20k noisy rows < 3 s, brute-force fuzz.
+- [FIX] [S2.2] — Any negative gain raises, all-zero gains undefined, string inputs rejected.
+- [FEAT] [S2.1] — Green.
+- [FEAT] [S2.2] — Green.
+- [FIX] [A2.3] — Verifier findings: extractor v2 (version stamped on run, stale versions rejected), prompts hashed by the version check (manifest rebuilt for the new hash format, no behaviour change to generate/validate), per-item transport errors, colon-free case ids, B11 add/drop guard (D-32), coverage as a ScoreResult with details, aligned ClaimSet.
+- [FEAT] [A2.3] — Green.
+- [FEAT] [A2.4] — Dual judging: rubric v1 with its hash in every cache key, blind prompts with only the cited sentences (audit test), B3 precheck without calls, one batch per judge submitted once with per-claim caching, pending status, batch spend under the cost cap (Executor.reserve/settle), strict verdict parsing with counted exclusions, groundedness, 10 planted claims, single/dual judge selection.
+- [FEAT] [S2.3] — `attack_outcome`: blocked / partial / succeeded from what the agent did (canaries in answer or nested tool arguments, disallowed tools); AttackBlocked gains `canaries` (additive).
+- [FEAT] [S2.3] — Green.
+- [MEASURE] [A2.1] — Real smoke run (20 cases × 4 models): 80/80 succeeded, $0.2418 spent; measured tokens put a 200-case run at $8.41. Spend ledger started at docs/measurements/spend-ledger.jsonl.

evalkit_harness-0.1.0/pyproject.toml

@@ -0,0 +1,94 @@
+[project]
+name = "evalkit-harness"
+version = "0.1.0"
+description = "Evaluate your own task across Claude and OpenAI models: grounded, calibrated, cost-aware, with confidence intervals."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Biplob Chakraborty" }]
+requires-python = ">=3.11"
+dependencies = [
+    "pydantic>=2.7",
+    "pyyaml>=6",
+]
+keywords = ["llm", "evaluation", "llm-as-judge", "groundedness", "benchmark", "agents"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[project.urls]
+Homepage = "https://github.com/biplob819/evalkit"
+Issues = "https://github.com/biplob819/evalkit/issues"
+
+[project.scripts]
+evalkit = "evalkit.cli:main"
+
+[project.optional-dependencies]
+anthropic = [
+    "anthropic>=1",
+]
+openai = [
+    "openai>=2",
+]
+agents = [
+    "duckdb>=1.1",
+    "pytz>=2026.6",
+]
+demo = [
+    "psycopg[binary]>=3.2",
+    "streamlit>=1.40",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=8",
+    "ruff>=0.6",
+    "pyright>=1.1.380",
+    "types-pyyaml>=6.0.12.20261113",
+]
+
+[build-system]
+requires = ["hatchling>=1.26"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/evalkit"]
+
+[tool.hatch.build.targets.sdist]
+# ship the library only; the build harness, specs and design docs live in the repo
+include = ["src/evalkit", "README.md", "LICENSE", "changelog.md"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+src = ["src", "scripts", "tests"]
+extend-exclude = ["*.md"]  # docs code blocks are illustrative; keep their hand alignment
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"]
+ignore = ["RUF001", "RUF002", "RUF003"]  # allow the › separator used in proof cells
+
+[tool.pyright]
+include = ["src", "scripts", "tests", "demo"]
+pythonVersion = "3.11"
+typeCheckingMode = "standard"
+extraPaths = ["scripts"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["scripts"]
+markers = [
+    "contract: schema, CLI and public API contract tests",
+    "golden: end-to-end run against recorded responses",
+    "judge: judge regression set",
+    "safety: secrets, blinding and guard tests",
+    "onboarding: no-key onboarding flow",
+]

evalkit_harness-0.1.0/src/evalkit/__init__.py

@@ -0,0 +1,5 @@
+"""EvalKit: evaluate your own task across Claude and OpenAI models."""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

evalkit_harness-0.1.0/src/evalkit/adapters/__init__.py

@@ -0,0 +1,25 @@
+"""Provider adapters. Import providers lazily: the SDKs are optional extras."""
+
+from evalkit.adapters.base import (
+    ALLOWED_PARAMS,
+    BatchRequest,
+    BatchResult,
+    CompletionRequest,
+    ModelAdapter,
+    RawCompletion,
+    discover,
+    get_adapter,
+    register,
+)
+
+__all__ = [
+    "ALLOWED_PARAMS",
+    "BatchRequest",
+    "BatchResult",
+    "CompletionRequest",
+    "ModelAdapter",
+    "RawCompletion",
+    "discover",
+    "get_adapter",
+    "register",
+]

evalkit_harness-0.1.0/src/evalkit/adapters/anthropic.py

@@ -0,0 +1,179 @@
+"""Anthropic Messages and Message Batches adapter (ADP-01, ADP-03).
+
+- No sampling parameters (D-22), no refusal fallbacks (D-23), effort set explicitly from
+  request params (D-24), JSON via `output_config.format` (D-24).
+- `output_tokens` includes thinking; thinking is reported separately as tokens_reasoning.
+"""
+
+from __future__ import annotations
+
+import inspect
+import time
+from collections.abc import Mapping
+from typing import Any
+
+from evalkit.adapters.base import (
+    BatchRequest,
+    BatchResult,
+    CompletionRequest,
+    ModelAdapter,
+    RawCompletion,
+    check_params,
+    check_unique_ids,
+    register,
+)
+from evalkit.cost import Price, call_cost, load_prices, price_for
+from evalkit.ratelimit import TokenBucket
+from evalkit.retry import with_retries
+from evalkit.runtime.canonical import to_jsonable
+
+
+@register("anthropic")
+class AnthropicAdapter(ModelAdapter):
+    provider = "anthropic"
+
+    require_effort = True
+
+    def __init__(
+        self,
+        client: Any = None,
+        prices: dict[str, Price] | None = None,
+        bucket: TokenBucket | None = None,
+        **retry: Any,
+    ):
+        self._client = client
+        self._prices = prices if prices is not None else load_prices()
+        self._bucket = bucket  # optional requests-per-second pacing (ADP-07)
+        self._retry = retry
+
+    async def _paced(self, call: Any) -> Any:
+        if self._bucket is not None:
+            await self._bucket.acquire()
+        return await with_retries(call, **self._retry)
+
+    @property
+    def client(self) -> Any:
+        if self._client is None:
+            import anthropic  # optional dependency, imported only when used
+
+            self._client = anthropic.AsyncAnthropic(max_retries=0)
+        return self._client
+
+    def build_params(self, req: CompletionRequest) -> dict[str, Any]:
+        check_params(req, require_effort=self.require_effort)
+        params: dict[str, Any] = {
+            "model": req.model_id,
+            "max_tokens": req.max_output_tokens,
+            "messages": [{"role": "user", "content": req.prompt}],
+        }
+        if req.turns:
+            params["messages"] = [{"role": r, "content": t} for r, t in req.turns]
+        if req.cache:
+            # automatic caching: the breakpoint follows the last block as the history grows
+            params["cache_control"] = {"type": "ephemeral"}
+        if req.system:
+            params["system"] = req.system
+        output_config: dict[str, Any] = {}
+        if "effort" in req.params:
+            output_config["effort"] = req.params["effort"]
+        if req.json_schema is not None:
+            output_config["format"] = {
+                "type": "json_schema",
+                "schema": to_jsonable(req.json_schema),
+            }
+        if output_config:
+            params["output_config"] = output_config
+        return params
+
+    def parse(
+        self, message: Any, req: CompletionRequest, latency_ms: int, *, batch: bool = False
+    ) -> RawCompletion:
+        text = "".join(b.text for b in message.content if getattr(b, "type", "") == "text")
+        u = message.usage
+        cache_read = getattr(u, "cache_read_input_tokens", 0) or 0
+        cache_write = getattr(u, "cache_creation_input_tokens", 0) or 0
+        details = getattr(u, "output_tokens_details", None)
+        thinking = (getattr(details, "thinking_tokens", 0) or 0) if details else 0
+        thinking = min(thinking, u.output_tokens)  # never bill more reasoning than output
+        tokens_in = u.input_tokens + cache_read + cache_write
+        tokens_out = max(u.output_tokens - thinking, 0)
+        cost = call_cost(
+            price_for(self._prices, req.model_id),
+            tokens_in=tokens_in,
+            tokens_out=tokens_out,
+            tokens_reasoning=thinking,
+            tokens_cached=cache_read,
+            tokens_cache_write=cache_write,
+            batch=batch,
+        )
+        stop = getattr(message, "stop_reason", "") or ""
+        return RawCompletion(
+            model=req.model,
+            model_snapshot=getattr(message, "model", "") or "",
+            params=dict(req.params),
+            raw_text=text,
+            tokens_in=tokens_in,
+            tokens_out=tokens_out,
+            tokens_reasoning=thinking,
+            tokens_cached=cache_read,
+            tokens_cache_write=cache_write,
+            latency_ms=latency_ms,
+            cost_usd=cost,
+            request_id=getattr(message, "_request_id", None) or getattr(message, "id", None),
+            stop_reason=stop,
+            refusal=stop == "refusal",
+            batch=batch,
+        )
+
+    async def complete(self, req: CompletionRequest) -> RawCompletion:
+        params = self.build_params(req)
+        t0 = time.perf_counter()
+        message = await self._paced(lambda: self.client.messages.create(**params))
+        return self.parse(message, req, int((time.perf_counter() - t0) * 1000))
+
+    async def submit_batch(self, reqs: list[BatchRequest]) -> str:
+        check_unique_ids(reqs)
+        requests = [
+            {"custom_id": r.custom_id, "params": self.build_params(r.request)} for r in reqs
+        ]
+        batch = await self._paced(lambda: self.client.messages.batches.create(requests=requests))
+        return str(batch.id)
+
+    async def collect_batch(
+        self, batch_id: str, reqs: Mapping[str, CompletionRequest]
+    ) -> list[BatchResult] | None:
+        status = await with_retries(
+            lambda: self.client.messages.batches.retrieve(batch_id), **self._retry
+        )
+        if status.processing_status != "ended":
+            return None
+        results = self.client.messages.batches.results(batch_id)
+        if inspect.isawaitable(results):
+            results = await results
+        out: list[BatchResult] = []
+        seen: set[str] = set()
+        async for item in results:
+            cid = item.custom_id
+            seen.add(cid)
+            if cid not in reqs:
+                continue  # not ours; results arrive in any order and are keyed by custom_id
+            kind = item.result.type
+            if kind == "succeeded":
+                out.append(
+                    BatchResult(
+                        custom_id=cid,
+                        completion=self.parse(
+                            item.result.message, reqs[cid], latency_ms=0, batch=True
+                        ),
+                    )
+                )
+            else:
+                err = getattr(item.result, "error", None)
+                # SDK shape: result.error is an ErrorResponse whose .error carries the type.
+                detail = getattr(getattr(err, "error", None), "type", None) or getattr(
+                    err, "type", None
+                )
+                out.append(BatchResult(custom_id=cid, error=f"{kind}:{detail}" if detail else kind))
+        for cid in sorted(set(reqs) - seen):
+            out.append(BatchResult(custom_id=cid, error="missing_from_results"))
+        return out