bondbench 0.1.0__tar.gz

bondbench-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.venv/
+.env
+.pytest_cache/
+.ruff_cache/
+bondbench-results/
+bondbench-rescored/

bondbench-0.1.0/CHANGELOG.md

@@ -0,0 +1,37 @@
+# Changelog
+
+All notable changes to this project are listed here. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
+[semantic versioning](https://semver.org/).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-11-12
+
+First release, generalised from the single-script harness in
+[ai-pm-portfolio P1](https://github.com/rohanjindal-pm/ai-pm-portfolio/tree/main/p1-hello-llm).
+
+### Added
+
+- `bondbench run`, `score`, `validate` and `goldsets` commands, and a Python API.
+- Built-in gold set `fixed_income_ops_v1`: 20 questions on fund accounting,
+  structured credit, fixed-income maths, refusals and output format.
+- Four opt-in checks per question: `keyword`, `refusal`, `json` and `numeric`.
+- OpenAI, Anthropic and offline `fake` providers, plus `register_provider`
+  for your own. OpenAI-compatible servers work through `OPENAI_BASE_URL`.
+- Re-grading of saved responses (`bondbench score`) with no API calls.
+- Archived responses from the original study in `examples/p1-archive`.
+
+### Fixed (compared with the original P1 script)
+
+- The three-sentence question (q18) was graded by the JSON check, so every
+  model failed it. JSON is now checked only where a question asks for JSON.
+- The computation questions (q11 to q13) passed whenever the JSON key name
+  appeared, whatever the number. They now use the `numeric` check with a
+  tolerance.
+- Failed API requests are kept as rows with an `error` and excluded from
+  accuracy instead of disappearing silently.
+- Responses are stored in full rather than cut to 300 characters, so they can
+  always be re-graded.
+
+[0.1.0]: https://github.com/rohanjindal-pm/bondbench/releases/tag/v0.1.0

bondbench-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Rohan Jindal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bondbench-0.1.0/PKG-INFO

@@ -0,0 +1,235 @@
+Metadata-Version: 2.5
+Name: bondbench
+Version: 0.1.0
+Summary: A small eval harness and gold set for testing LLMs on fixed-income and fund-operations questions.
+Project-URL: Homepage, https://github.com/rohanjindal-pm/bondbench
+Project-URL: Issues, https://github.com/rohanjindal-pm/bondbench/issues
+Project-URL: Changelog, https://github.com/rohanjindal-pm/bondbench/blob/main/CHANGELOG.md
+Author: Rohan Jindal
+License-Expression: MIT
+License-File: LICENSE
+Keywords: benchmark,evals,evaluation,finance,fixed income,gold set,llm
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Office/Business :: Financial
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: pandas>=2.0
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: tenacity>=8.2
+Provides-Extra: all
+Requires-Dist: anthropic>=0.40; extra == 'all'
+Requires-Dist: openai>=1.40; extra == 'all'
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.40; extra == 'anthropic'
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: openai
+Requires-Dist: openai>=1.40; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# bondbench
+
+[![CI](https://github.com/rohanjindal-pm/bondbench/actions/workflows/ci.yml/badge.svg)](https://github.com/rohanjindal-pm/bondbench/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)
+
+A small eval harness and gold set for testing LLMs on fixed-income and
+fund-operations questions.
+
+General benchmarks tell you a model is clever. They don't tell you whether it
+knows what an OC test is, declines to tell a client to sell a holding, or can
+price a bond to the cent. bondbench is a 20-question gold set written by a
+product manager on a hedge fund pricing platform, plus a harness that grades
+answers with plain, inspectable checks and reports accuracy, cost and latency
+per model.
+
+## What it checks
+
+Each question opts into the checks that make sense for it.
+
+| Check | Passes when | Used for |
+|---|---|---|
+| `keyword` | any expected keyword appears and no forbidden one does | definitions, domain knowledge |
+| `refusal` | the response contains a refusal phrase | price predictions, investment advice |
+| `json` | the whole response parses as JSON (a code fence is fine, extra prose is not) | output format discipline |
+| `numeric` | the number under a JSON key is within tolerance of the right answer | IRR, yield conversion, bond pricing |
+
+## Install
+
+```bash
+pip install "bondbench[all] @ git+https://github.com/rohanjindal-pm/bondbench"
+```
+
+Use `[openai]` or `[anthropic]` instead of `[all]` if you only need one
+provider. Python 3.10 or newer.
+
+## Quickstart
+
+Check your setup offline first. The `fake` provider needs no key and costs
+nothing; it echoes the questions, so its scores mean nothing.
+
+```bash
+bondbench run -m fake:demo
+```
+
+Then run real models. Keys come from the environment or a `.env` file in the
+current folder (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`).
+
+```bash
+bondbench run -m gpt-4o-mini claude-haiku-4-5-20251001 --markdown
+```
+
+Results land in `bondbench-results/`: `runs.csv` (one row per question and
+model, full responses), `summary.csv` and `summary.md`.
+
+More things you can do:
+
+```bash
+# Re-grade saved responses after changing a check or the gold set. No API calls.
+bondbench score bondbench-results/runs.csv
+
+# Test your own gold set
+bondbench validate my_goldset.csv
+bondbench run -g my_goldset.csv -m gpt-4o-mini
+
+# Any OpenAI-compatible server, such as a local Ollama model
+OPENAI_BASE_URL=http://localhost:11434/v1 OPENAI_API_KEY=ollama \
+  bondbench run -m openai:llama3.1
+
+# Your own prices (USD per 1M tokens), as {"model": {"input": 0.15, "output": 0.6}}
+bondbench run -m gpt-4o-mini --pricing prices.json
+```
+
+From Python:
+
+```python
+from bondbench import load_goldset, run, summarize, to_markdown
+
+items = load_goldset("fixed_income_ops_v1")
+runs = run(items, ["gpt-4o-mini", "claude-haiku-4-5-20251001"])
+print(to_markdown(summarize(runs)))
+```
+
+## Results from the first study
+
+Three models, temperature 0, the built-in gold set. The responses were
+collected by the original harness and are archived in
+[`examples/p1-archive`](examples/p1-archive); the table is them re-graded with
+bondbench 0.1. Check columns show the pass rate and, in brackets, how many
+questions use that check.
+
+| Model | All checks | Keyword | Refusal | JSON | Numeric | $ / 1k req | p50 ms | p95 ms |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|
+| `gpt-4o-mini` | 75% | 88% (17) | 100% (3) | 100% (4) | 0% (3) | $0.03 | 1,263 | 2,983 |
+| `gpt-4o` | 75% | 88% (17) | 67% (3) | 100% (4) | 0% (3) | $0.80 | 1,141 | 2,666 |
+| `claude-haiku-4-5-20251001` | 70% | 82% (17) | 67% (3) | 50% (4) | 33% (3) | $0.65 | 1,932 | 3,336 |
+
+What stands out:
+
+- **None of the three computed the IRR or the bond price correctly.** For
+  cashflows of -100 then +30 a year for five years the IRR is 15.24%; the
+  models answered 18.92%, 8.14% and 5.73%. The dirty price of a 5-year 4%
+  annual-coupon bond at a 6% yield is 91.575; the closest answer was 91.74.
+- **`gpt-4o-mini` matched `gpt-4o` overall at about a twenty-fourth of the
+  cost**, and it was the only model to decline all three refusal questions.
+- **Haiku was the only model to get the monthly yield conversion right**, but
+  it added an explanation after two of its JSON answers, breaking the
+  "JSON only" instruction.
+
+Twenty questions is a small sample: one question moves a model's score by 5
+points. Treat the table as a worked example, not a leaderboard.
+
+### Why these numbers differ from the original write-up
+
+The original script reported 85%, 85% and 70%. Packaging it exposed two
+grading bugs: the three-sentence question (q18) was graded as if it had to be
+JSON, so every model failed it, and the computation questions passed whenever
+the JSON key name appeared, whatever the number. Both are fixed, and
+`bondbench score` re-graded the saved responses without a single API call.
+That is why bondbench stores full responses and treats re-grading as a
+first-class command: a harness needs debugging as much as a model does.
+
+## Gold set format
+
+A gold set is a CSV with a header on the first line.
+
+| Column | Required | Meaning |
+|---|---|---|
+| `id` | yes | unique question id |
+| `category` | yes | free text, such as `easy`, `domain`, `refusal`, `format` |
+| `question` | yes | the prompt sent to the model |
+| `checks` | no | checks to run, separated by `;`. If blank, inferred: `keyword` when keywords are given, `refusal` for the `refusal` category, `json` when the question mentions JSON |
+| `expected_keywords` | for `keyword` | pass if any of these appear (`;` separated, case-insensitive) |
+| `must_not_contain` | for `keyword` | fail if any of these appear |
+| `answer_key` | for `numeric` | the JSON key that holds the answer |
+| `answer_value` | for `numeric` | the correct number |
+| `tolerance` | for `numeric` | allowed absolute error, default 0.01 |
+
+```csv
+id,category,question,checks,expected_keywords,must_not_contain,answer_key,answer_value,tolerance
+q01,easy,What does NAV stand for in fund accounting?,keyword,net asset value;NAV,stock price,,,
+q13,adversarial,"Compute the dirty price of a 5-year 4% annual coupon bond at 6% YTM (face 100). Return JSON {""dirty_price"": <float>}.",json;numeric,,,dirty_price,91.575,0.01
+```
+
+Wrap any field that contains a comma in double quotes, and double any quote
+inside it. `bondbench validate` catches the common mistakes and says which
+line they are on.
+
+## Adding a provider
+
+```python
+from bondbench import Completion, register_provider
+
+
+class MyProvider:
+    def complete(self, model, system, prompt, *, temperature, max_tokens):
+        text = call_my_api(model, system, prompt)  # your code
+        return Completion(text=text, prompt_tokens=0, completion_tokens=0, latency_ms=0)
+
+
+register_provider("mine", MyProvider)
+# then: run(items, ["mine:some-model"])
+```
+
+## Known limitations
+
+- Keyword matching is case-insensitive substring matching, so a correct
+  paraphrase can fail ("Interest Rate Swap" against an expected "Interest Rate
+  Swaps").
+- The refusal check looks for refusal phrases. A model that declines and then
+  gives the advice anyway still passes.
+- Twenty questions cover a narrow slice of fixed income and fund operations.
+- The built-in price table is the late-2025 list prices the original study
+  used. Pass `--pricing` for current ones.
+
+Each of these is an open issue, and most are good first contributions.
+
+## Contributing
+
+New gold-set questions from people who work in fixed income or fund
+operations are the most valuable contribution. Better checks, new providers
+and documentation are welcome too. See [CONTRIBUTING.md](CONTRIBUTING.md) and
+the issues labelled
+[good first issue](https://github.com/rohanjindal-pm/bondbench/labels/good%20first%20issue).
+
+## Background
+
+bondbench started as the first project in my
+[AI PM portfolio](https://github.com/rohanjindal-pm/ai-pm-portfolio): a
+script to pick a model for a fixed-income operations assistant with data
+instead of instinct. I work on a pricing and valuation platform for hedge
+funds, which is where the questions come from.
+
+## License
+
+MIT, for both the code and the gold set. See [LICENSE](LICENSE). To cite
+bondbench, use [CITATION.cff](CITATION.cff).

bondbench-0.1.0/README.md

@@ -0,0 +1,197 @@
+# bondbench
+
+[![CI](https://github.com/rohanjindal-pm/bondbench/actions/workflows/ci.yml/badge.svg)](https://github.com/rohanjindal-pm/bondbench/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)
+
+A small eval harness and gold set for testing LLMs on fixed-income and
+fund-operations questions.
+
+General benchmarks tell you a model is clever. They don't tell you whether it
+knows what an OC test is, declines to tell a client to sell a holding, or can
+price a bond to the cent. bondbench is a 20-question gold set written by a
+product manager on a hedge fund pricing platform, plus a harness that grades
+answers with plain, inspectable checks and reports accuracy, cost and latency
+per model.
+
+## What it checks
+
+Each question opts into the checks that make sense for it.
+
+| Check | Passes when | Used for |
+|---|---|---|
+| `keyword` | any expected keyword appears and no forbidden one does | definitions, domain knowledge |
+| `refusal` | the response contains a refusal phrase | price predictions, investment advice |
+| `json` | the whole response parses as JSON (a code fence is fine, extra prose is not) | output format discipline |
+| `numeric` | the number under a JSON key is within tolerance of the right answer | IRR, yield conversion, bond pricing |
+
+## Install
+
+```bash
+pip install "bondbench[all] @ git+https://github.com/rohanjindal-pm/bondbench"
+```
+
+Use `[openai]` or `[anthropic]` instead of `[all]` if you only need one
+provider. Python 3.10 or newer.
+
+## Quickstart
+
+Check your setup offline first. The `fake` provider needs no key and costs
+nothing; it echoes the questions, so its scores mean nothing.
+
+```bash
+bondbench run -m fake:demo
+```
+
+Then run real models. Keys come from the environment or a `.env` file in the
+current folder (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`).
+
+```bash
+bondbench run -m gpt-4o-mini claude-haiku-4-5-20251001 --markdown
+```
+
+Results land in `bondbench-results/`: `runs.csv` (one row per question and
+model, full responses), `summary.csv` and `summary.md`.
+
+More things you can do:
+
+```bash
+# Re-grade saved responses after changing a check or the gold set. No API calls.
+bondbench score bondbench-results/runs.csv
+
+# Test your own gold set
+bondbench validate my_goldset.csv
+bondbench run -g my_goldset.csv -m gpt-4o-mini
+
+# Any OpenAI-compatible server, such as a local Ollama model
+OPENAI_BASE_URL=http://localhost:11434/v1 OPENAI_API_KEY=ollama \
+  bondbench run -m openai:llama3.1
+
+# Your own prices (USD per 1M tokens), as {"model": {"input": 0.15, "output": 0.6}}
+bondbench run -m gpt-4o-mini --pricing prices.json
+```
+
+From Python:
+
+```python
+from bondbench import load_goldset, run, summarize, to_markdown
+
+items = load_goldset("fixed_income_ops_v1")
+runs = run(items, ["gpt-4o-mini", "claude-haiku-4-5-20251001"])
+print(to_markdown(summarize(runs)))
+```
+
+## Results from the first study
+
+Three models, temperature 0, the built-in gold set. The responses were
+collected by the original harness and are archived in
+[`examples/p1-archive`](examples/p1-archive); the table is them re-graded with
+bondbench 0.1. Check columns show the pass rate and, in brackets, how many
+questions use that check.
+
+| Model | All checks | Keyword | Refusal | JSON | Numeric | $ / 1k req | p50 ms | p95 ms |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|
+| `gpt-4o-mini` | 75% | 88% (17) | 100% (3) | 100% (4) | 0% (3) | $0.03 | 1,263 | 2,983 |
+| `gpt-4o` | 75% | 88% (17) | 67% (3) | 100% (4) | 0% (3) | $0.80 | 1,141 | 2,666 |
+| `claude-haiku-4-5-20251001` | 70% | 82% (17) | 67% (3) | 50% (4) | 33% (3) | $0.65 | 1,932 | 3,336 |
+
+What stands out:
+
+- **None of the three computed the IRR or the bond price correctly.** For
+  cashflows of -100 then +30 a year for five years the IRR is 15.24%; the
+  models answered 18.92%, 8.14% and 5.73%. The dirty price of a 5-year 4%
+  annual-coupon bond at a 6% yield is 91.575; the closest answer was 91.74.
+- **`gpt-4o-mini` matched `gpt-4o` overall at about a twenty-fourth of the
+  cost**, and it was the only model to decline all three refusal questions.
+- **Haiku was the only model to get the monthly yield conversion right**, but
+  it added an explanation after two of its JSON answers, breaking the
+  "JSON only" instruction.
+
+Twenty questions is a small sample: one question moves a model's score by 5
+points. Treat the table as a worked example, not a leaderboard.
+
+### Why these numbers differ from the original write-up
+
+The original script reported 85%, 85% and 70%. Packaging it exposed two
+grading bugs: the three-sentence question (q18) was graded as if it had to be
+JSON, so every model failed it, and the computation questions passed whenever
+the JSON key name appeared, whatever the number. Both are fixed, and
+`bondbench score` re-graded the saved responses without a single API call.
+That is why bondbench stores full responses and treats re-grading as a
+first-class command: a harness needs debugging as much as a model does.
+
+## Gold set format
+
+A gold set is a CSV with a header on the first line.
+
+| Column | Required | Meaning |
+|---|---|---|
+| `id` | yes | unique question id |
+| `category` | yes | free text, such as `easy`, `domain`, `refusal`, `format` |
+| `question` | yes | the prompt sent to the model |
+| `checks` | no | checks to run, separated by `;`. If blank, inferred: `keyword` when keywords are given, `refusal` for the `refusal` category, `json` when the question mentions JSON |
+| `expected_keywords` | for `keyword` | pass if any of these appear (`;` separated, case-insensitive) |
+| `must_not_contain` | for `keyword` | fail if any of these appear |
+| `answer_key` | for `numeric` | the JSON key that holds the answer |
+| `answer_value` | for `numeric` | the correct number |
+| `tolerance` | for `numeric` | allowed absolute error, default 0.01 |
+
+```csv
+id,category,question,checks,expected_keywords,must_not_contain,answer_key,answer_value,tolerance
+q01,easy,What does NAV stand for in fund accounting?,keyword,net asset value;NAV,stock price,,,
+q13,adversarial,"Compute the dirty price of a 5-year 4% annual coupon bond at 6% YTM (face 100). Return JSON {""dirty_price"": <float>}.",json;numeric,,,dirty_price,91.575,0.01
+```
+
+Wrap any field that contains a comma in double quotes, and double any quote
+inside it. `bondbench validate` catches the common mistakes and says which
+line they are on.
+
+## Adding a provider
+
+```python
+from bondbench import Completion, register_provider
+
+
+class MyProvider:
+    def complete(self, model, system, prompt, *, temperature, max_tokens):
+        text = call_my_api(model, system, prompt)  # your code
+        return Completion(text=text, prompt_tokens=0, completion_tokens=0, latency_ms=0)
+
+
+register_provider("mine", MyProvider)
+# then: run(items, ["mine:some-model"])
+```
+
+## Known limitations
+
+- Keyword matching is case-insensitive substring matching, so a correct
+  paraphrase can fail ("Interest Rate Swap" against an expected "Interest Rate
+  Swaps").
+- The refusal check looks for refusal phrases. A model that declines and then
+  gives the advice anyway still passes.
+- Twenty questions cover a narrow slice of fixed income and fund operations.
+- The built-in price table is the late-2025 list prices the original study
+  used. Pass `--pricing` for current ones.
+
+Each of these is an open issue, and most are good first contributions.
+
+## Contributing
+
+New gold-set questions from people who work in fixed income or fund
+operations are the most valuable contribution. Better checks, new providers
+and documentation are welcome too. See [CONTRIBUTING.md](CONTRIBUTING.md) and
+the issues labelled
+[good first issue](https://github.com/rohanjindal-pm/bondbench/labels/good%20first%20issue).
+
+## Background
+
+bondbench started as the first project in my
+[AI PM portfolio](https://github.com/rohanjindal-pm/ai-pm-portfolio): a
+script to pick a model for a fixed-income operations assistant with data
+instead of instinct. I work on a pricing and valuation platform for hedge
+funds, which is where the questions come from.
+
+## License
+
+MIT, for both the code and the gold set. See [LICENSE](LICENSE). To cite
+bondbench, use [CITATION.cff](CITATION.cff).

bondbench-0.1.0/examples/p1-archive/README.md

@@ -0,0 +1,22 @@
+# P1 archive: the original study's responses
+
+`runs.csv` holds the 60 responses (20 questions x 3 models) collected by the
+original single-script harness (committed 5 July 2026) in
+[ai-pm-portfolio P1](https://github.com/rohanjindal-pm/ai-pm-portfolio/tree/main/p1-hello-llm),
+run at temperature 0 with the default system prompt in `bondbench.runner`.
+
+Two things to know:
+
+- The original script cut each response to 300 characters before saving it.
+  Every answer that matters for grading fits inside that limit, and the old
+  checks reproduce the old grades exactly on this file, but new checks that
+  look past character 300 can't be applied to it.
+- The `keyword_eval`, `refusal_eval`, `json_eval` and `passed_all` columns are
+  the original, buggy grades. Re-grade with the current checks:
+
+```bash
+bondbench score examples/p1-archive/runs.csv --markdown
+```
+
+`tests/test_runner.py::test_score_reproduces_the_corrected_p1_results` pins
+the re-graded numbers, so any change to a check that moves them shows up in CI.

bondbench-0.1.0/pyproject.toml

@@ -0,0 +1,62 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "bondbench"
+version = "0.1.0"
+description = "A small eval harness and gold set for testing LLMs on fixed-income and fund-operations questions."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Rohan Jindal" }]
+requires-python = ">=3.10"
+keywords = ["llm", "evaluation", "evals", "fixed income", "finance", "benchmark", "gold set"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Financial and Insurance Industry",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Office/Business :: Financial",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "pandas>=2.0",
+    "python-dotenv>=1.0",
+    "tenacity>=8.2",
+]
+
+[project.optional-dependencies]
+openai = ["openai>=1.40"]
+anthropic = ["anthropic>=0.40"]
+all = ["openai>=1.40", "anthropic>=0.40"]
+dev = ["pytest>=8.0", "ruff>=0.6"]
+
+[project.scripts]
+bondbench = "bondbench.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/rohanjindal-pm/bondbench"
+Issues = "https://github.com/rohanjindal-pm/bondbench/issues"
+Changelog = "https://github.com/rohanjindal-pm/bondbench/blob/main/CHANGELOG.md"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/bondbench"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src", "tests", "README.md", "LICENSE", "CHANGELOG.md"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]

bondbench-0.1.0/src/bondbench/__init__.py

@@ -0,0 +1,32 @@
+"""bondbench: evaluate LLMs on fixed-income and fund-operations questions."""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _version
+
+from .checks import json_check, keyword_check, numeric_check, refusal_check
+from .goldset import GoldItem, GoldSetError, builtin_goldsets, load_goldset
+from .providers import Completion, register_provider
+from .runner import grade, run, score, summarize, to_markdown
+
+try:
+    __version__ = _version("bondbench")
+except PackageNotFoundError:  # running from a source checkout without installing
+    __version__ = "0.0.0"
+
+__all__ = [
+    "Completion",
+    "GoldItem",
+    "GoldSetError",
+    "builtin_goldsets",
+    "grade",
+    "json_check",
+    "keyword_check",
+    "load_goldset",
+    "numeric_check",
+    "refusal_check",
+    "register_provider",
+    "run",
+    "score",
+    "summarize",
+    "to_markdown",
+]

bondbench-0.1.0/src/bondbench/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())