banna 0.1.0__tar.gz

banna-0.1.0/PKG-INFO

@@ -0,0 +1,225 @@
+Metadata-Version: 2.4
+Name: banna
+Version: 0.1.0
+Summary: A from-scratch, provider-agnostic reasoning agent with a typed state substrate and verifier-guided search. Primary benchmark: GAIA.
+Author-email: Siavash Monfared <monfared@alum.mit.edu>
+License: MIT
+Project-URL: Homepage, https://github.com/siavashmonfared/banna
+Project-URL: Repository, https://github.com/siavashmonfared/banna
+Project-URL: Issues, https://github.com/siavashmonfared/banna/issues
+Keywords: agent,llm,react,verifier,gaia,reasoning,anthropic,openai,gemini
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: anthropic>=0.40
+Requires-Dist: openai>=1.50
+Requires-Dist: google-generativeai>=0.8
+Requires-Dist: requests>=2.31
+Requires-Dist: pydantic>=2.6
+Requires-Dist: datasets>=2.19
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: pypdf>=4.0
+Requires-Dist: openpyxl>=3.1
+Requires-Dist: pandas>=2.2
+Requires-Dist: pillow>=10.0
+Requires-Dist: rich>=13.7
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: ruff>=0.5; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Provides-Extra: pdf
+Requires-Dist: pdfplumber>=0.11; extra == "pdf"
+
+# banna
+
+A from-scratch, provider-agnostic reasoning agent with a **typed state substrate** and a **verifier-guided** loop. Built to study where ReAct-style agents fail on the **GAIA** benchmark and to fix those failures structurally — not with prompt patches.
+
+No LangChain, no LlamaIndex, no smolagents in the core. The reasoning loop is a typed transition function over `(state, action, observation) → state'`; ReAct, verifier-retry, planner-ReAct, BFS/DFS/best-first-over-plans, and best-of-N are each ~200 LOC `Policy` implementations over that same substrate.
+
+## What's interesting about this repo
+
+1. **Forensic GAIA debugging.** A full-validation run on `gpt-5-nano` was instrumented end-to-end, traces were dumped per task, and seven distinct structural failure modes were diagnosed and fixed — not by prompt-tweaking, but by changing the loop. See [Failure modes & fixes](#failure-modes--fixes-the-c1c6-pass) below.
+2. **Multi-axis budget tracker** that separates *productive* steps from *repair* steps. A model stuck in an `[empty_reply]` loop no longer burns its productive-step budget; instead it trips a separate `max_repair_steps` axis with a forced tool-choice escape.
+3. **Per-verifier actionable nudges.** Each verifier (Arithmetic, Citation, Coverage, Format) attaches a `meta["nudge"]` to its fail verdicts that names the missing thing (the recomputed value, the missing evidence_id, the unsupported number, the empty field). The retry policy groups these by verifier and emits one line per kind — short enough that the model actually reads them.
+4. **Budget-exhaustion synthesis.** When the agent runs out of steps mid-task, instead of returning `null`, a final forced-`final_answer` call gives it one last shot with a cheap fallback chain (last claim → last short text → none).
+5. **Provider-agnostic tool forcing.** A single helper translates "force any tool" into OpenAI's `tool_choice: "required"`, Anthropic's `{type: "any"}`, and Gemini's `ANY` mode — used to break out of empty-reply loops.
+
+## Architecture
+
+```mermaid
+flowchart TB
+    Q[User question<br/>+ optional attachment] --> AG[Agent.run_policy]
+
+    subgraph SUB[banna_agent runtime]
+      AG --> ST[AgentState<br/>typed: Trace · Evidence · Claim · Budget]
+      ST -->|propose| POL{Policy}
+
+      POL --> REACT[ReActPolicy]
+      POL --> VR[VerifierRetryPolicy<br/>wraps inner]
+      POL --> PL[PlannerReActPolicy]
+      POL --> BFS[BFS / DFS / Best-First<br/>over plans]
+      POL --> BON[BestOfNPolicy<br/>K trajectories + selector]
+
+      REACT -->|Action| EX[Execute]
+      VR -->|FINAL_ANSWER| VERS{Verifiers}
+      VERS -- pass --> COMMIT[commit]
+      VERS -- fail --> NUDGE[per-verifier nudge<br/>→ THINK feedback]
+      NUDGE --> POL
+
+      EX --> TOOLS[ToolRegistry]
+      TOOLS --> T1[search]
+      TOOLS --> T2[read_url]
+      TOOLS --> T3[read_file<br/>+ pdf / xlsx tools]
+      TOOLS --> T4[python_sandbox]
+      TOOLS --> T5[calculator]
+      TOOLS --> T6[run_shell · grep · list_files]
+
+      EX -->|observation| ST
+      ST --> BUD[BudgetTracker<br/>steps · repair_steps · wall · tokens · cost]
+      BUD -. trip .-> SYN[synthesize_on_exhaustion<br/>forced final_answer]
+      SYN --> COMMIT
+    end
+
+    subgraph VERIFIERS[Verifiers]
+      VF[FormatVerifier<br/>shape / empty answer]
+      VA[ArithmeticVerifier<br/>safe-AST recompute]
+      VC[CitationVerifier<br/>numeric + Jaccard support]
+      VG[CoverageVerifier<br/>claim ↔ evidence]
+    end
+    VERS --- VF
+    VERS --- VA
+    VERS --- VC
+    VERS --- VG
+
+    COMMIT --> ANS[answer]
+```
+
+## Install
+
+```bash
+# 1. From PyPI (once published)
+pip install banna
+
+# 2. From GitHub directly (no clone, no PyPI required)
+pip install git+https://github.com/siavashmonfared/banna.git
+
+# 3. Isolated install with pipx (recommended for CLI use)
+pipx install git+https://github.com/siavashmonfared/banna.git
+
+# 4. From a local clone (for development)
+git clone https://github.com/siavashmonfared/banna.git
+cd banna
+pip install -e ".[dev]"
+```
+
+Any install path drops a `banna` (and `banna-agent`) executable on your `$PATH`.
+
+## Quickstart
+
+```bash
+# set at least one provider key
+export OPENAI_API_KEY=sk-...        # or ANTHROPIC_API_KEY=... / GEMINI_API_KEY=...
+
+# open the interactive REPL
+banna --policy verifier_retry --provider openai --model gpt-5-nano
+
+# or run a single GAIA Level-1 question (no REPL)
+python -m banna_agent.benchmarks.gaia.runner \
+    --policy verifier_retry --provider openai --model gpt-5-nano \
+    --level 1 --n 1
+```
+
+### Example REPL session
+
+```
+$ banna --policy verifier_retry --provider openai --model gpt-5-nano
+
+● banna · v0.1.0   provider=openai   model=gpt-5-nano   policy=verifier_retry
+
+> How many studio albums did Mercedes Sosa release between 2000 and 2009?
+
+  thinking…
+  ▸ search(query="Mercedes Sosa discography studio albums 2000-2009")
+    ↳ 8 results · evidence_id ev_a3f
+  ▸ read_url(url="https://en.wikipedia.org/wiki/Mercedes_Sosa")
+    ↳ 12.4 kB · evidence_id ev_91c
+  thinking…
+  ▸ final_answer(answer="3", evidence_ids=["ev_a3f", "ev_91c"])
+  verifiers: format ✓  citation ✓  coverage ✓  arithmetic skip
+
+● banna
+  3
+
+  3 steps · 4.7s · 1840→210 tok · $0.0021
+
+> /show trace
+  …step-by-step dump of action + observation + meta…
+
+> /exit
+```
+
+The full GAIA validation runner (165 questions across L1/L2/L3) is in `experiments/02_gaia_full/run.py`.
+
+## Failure modes & fixes (the C1–C6 pass)
+
+Diagnosed from a full GAIA validation run on `gpt-5-nano`. Each fix lands as a structural change to the loop, with unit tests pinning the new behavior.
+
+| ID | Failure mode | Root cause | Fix |
+|----|--------------|------------|-----|
+| C1 | `[empty_reply]` loops eat the step budget | Repair-style THINKs counted as productive steps | New `Budget.repair_steps_used` axis + `max_repair_steps=6`; `meta["repair"]=True` routes off the main counter |
+| C2 | Model returns empty content + no tool call | No detection / no escape | After 2 consecutive empties with no evidence, force `tool_choice` to any tool (provider-agnostic) |
+| C3 | `pred_answer=null` on budget exhaustion | Loop exits with no commit | `policy.synthesize_on_exhaustion(state)`: one threaded LLM call with forced `final_answer` + cheap fallback chain |
+| C4 | L1 step cap too tight (8 steps) | Default budget profile | L1/L2/L3 caps bumped to 12/18/24 |
+| C5 | Rich file tools never used on attachments | Hint steered model toward `read_file` even for PDF/XLSX | Extension-routed `_file_hint()` + cheap `_file_summary()` (pypdf page count, openpyxl sheet names, CSV header) |
+| C6 | Verifier retries low repair rate | Feedback was generic | Each verifier populates `meta["nudge"]` with a verifier-specific actionable instruction; retry feedback groups by verifier |
+
+## GAIA validation results
+
+> The fixes are landed and tested (562 tests passing in this public repo, 568 in the private superset). The post-fix re-run on GAIA validation is the next thing on the queue.
+
+| Run | Provider · model | Policy | Set | Accuracy | Notes |
+|-----|------------------|--------|-----|----------|-------|
+| Pre-C1–C6 | OpenAI · `gpt-5-nano` | verifier_retry | GAIA val (165 Q) | **33.9 %** (56 / 165) | $0.94, 7 structural bugs surfaced |
+| Post-C1–C6 | OpenAI · `gpt-5-nano` | verifier_retry | GAIA val (165 Q) | _re-run pending_ | target ≥ 40 % |
+| Post-C1–C6 | OpenAI · `gpt-5-nano` | best_of_n (K=3) | GAIA val (165 Q) | _re-run pending_ | stretch ≥ 45 % |
+
+(Numbers populate once the re-run completes; CI is wired to gate on a held-out smoke subset.)
+
+## Repo layout
+
+```
+src/banna_agent/
+├── core/          AgentState, Trace, Action, Budget, EventLog, run_policy
+├── llm/           provider-agnostic LLMClient + adapters (anthropic, openai, gemini, ollama, bedrock)
+├── tools/         search, read_url, read_file, pdf/xlsx tools, python_sandbox,
+│                  calculator, run_shell, grep, list_files, plan, memory, final_answer
+├── policies/      react, planner_react, verifier_retry, bfs/dfs/best_first_over_plans, best_of_n
+├── verifiers/     arithmetic, citation, coverage, format, command (+ base protocol)
+├── benchmarks/    gaia/ (loader, runner, scorer, report)
+├── memory/        in_memory_store, jsonl_store, skill_library, embeddings
+└── cli/           Rich-based REPL: /policy /budget /show /skills /compact /save /load …
+```
+
+Tests live in `tests/` and are organized to mirror `src/`. Run them with:
+
+```bash
+pytest -q
+```
+
+Current status: **562 passed, 0 failed** on this public branch (no external substrate dependencies).
+
+## License
+
+MIT

banna-0.1.0/README.md

@@ -0,0 +1,181 @@
+# banna
+
+A from-scratch, provider-agnostic reasoning agent with a **typed state substrate** and a **verifier-guided** loop. Built to study where ReAct-style agents fail on the **GAIA** benchmark and to fix those failures structurally — not with prompt patches.
+
+No LangChain, no LlamaIndex, no smolagents in the core. The reasoning loop is a typed transition function over `(state, action, observation) → state'`; ReAct, verifier-retry, planner-ReAct, BFS/DFS/best-first-over-plans, and best-of-N are each ~200 LOC `Policy` implementations over that same substrate.
+
+## What's interesting about this repo
+
+1. **Forensic GAIA debugging.** A full-validation run on `gpt-5-nano` was instrumented end-to-end, traces were dumped per task, and seven distinct structural failure modes were diagnosed and fixed — not by prompt-tweaking, but by changing the loop. See [Failure modes & fixes](#failure-modes--fixes-the-c1c6-pass) below.
+2. **Multi-axis budget tracker** that separates *productive* steps from *repair* steps. A model stuck in an `[empty_reply]` loop no longer burns its productive-step budget; instead it trips a separate `max_repair_steps` axis with a forced tool-choice escape.
+3. **Per-verifier actionable nudges.** Each verifier (Arithmetic, Citation, Coverage, Format) attaches a `meta["nudge"]` to its fail verdicts that names the missing thing (the recomputed value, the missing evidence_id, the unsupported number, the empty field). The retry policy groups these by verifier and emits one line per kind — short enough that the model actually reads them.
+4. **Budget-exhaustion synthesis.** When the agent runs out of steps mid-task, instead of returning `null`, a final forced-`final_answer` call gives it one last shot with a cheap fallback chain (last claim → last short text → none).
+5. **Provider-agnostic tool forcing.** A single helper translates "force any tool" into OpenAI's `tool_choice: "required"`, Anthropic's `{type: "any"}`, and Gemini's `ANY` mode — used to break out of empty-reply loops.
+
+## Architecture
+
+```mermaid
+flowchart TB
+    Q[User question<br/>+ optional attachment] --> AG[Agent.run_policy]
+
+    subgraph SUB[banna_agent runtime]
+      AG --> ST[AgentState<br/>typed: Trace · Evidence · Claim · Budget]
+      ST -->|propose| POL{Policy}
+
+      POL --> REACT[ReActPolicy]
+      POL --> VR[VerifierRetryPolicy<br/>wraps inner]
+      POL --> PL[PlannerReActPolicy]
+      POL --> BFS[BFS / DFS / Best-First<br/>over plans]
+      POL --> BON[BestOfNPolicy<br/>K trajectories + selector]
+
+      REACT -->|Action| EX[Execute]
+      VR -->|FINAL_ANSWER| VERS{Verifiers}
+      VERS -- pass --> COMMIT[commit]
+      VERS -- fail --> NUDGE[per-verifier nudge<br/>→ THINK feedback]
+      NUDGE --> POL
+
+      EX --> TOOLS[ToolRegistry]
+      TOOLS --> T1[search]
+      TOOLS --> T2[read_url]
+      TOOLS --> T3[read_file<br/>+ pdf / xlsx tools]
+      TOOLS --> T4[python_sandbox]
+      TOOLS --> T5[calculator]
+      TOOLS --> T6[run_shell · grep · list_files]
+
+      EX -->|observation| ST
+      ST --> BUD[BudgetTracker<br/>steps · repair_steps · wall · tokens · cost]
+      BUD -. trip .-> SYN[synthesize_on_exhaustion<br/>forced final_answer]
+      SYN --> COMMIT
+    end
+
+    subgraph VERIFIERS[Verifiers]
+      VF[FormatVerifier<br/>shape / empty answer]
+      VA[ArithmeticVerifier<br/>safe-AST recompute]
+      VC[CitationVerifier<br/>numeric + Jaccard support]
+      VG[CoverageVerifier<br/>claim ↔ evidence]
+    end
+    VERS --- VF
+    VERS --- VA
+    VERS --- VC
+    VERS --- VG
+
+    COMMIT --> ANS[answer]
+```
+
+## Install
+
+```bash
+# 1. From PyPI (once published)
+pip install banna
+
+# 2. From GitHub directly (no clone, no PyPI required)
+pip install git+https://github.com/siavashmonfared/banna.git
+
+# 3. Isolated install with pipx (recommended for CLI use)
+pipx install git+https://github.com/siavashmonfared/banna.git
+
+# 4. From a local clone (for development)
+git clone https://github.com/siavashmonfared/banna.git
+cd banna
+pip install -e ".[dev]"
+```
+
+Any install path drops a `banna` (and `banna-agent`) executable on your `$PATH`.
+
+## Quickstart
+
+```bash
+# set at least one provider key
+export OPENAI_API_KEY=sk-...        # or ANTHROPIC_API_KEY=... / GEMINI_API_KEY=...
+
+# open the interactive REPL
+banna --policy verifier_retry --provider openai --model gpt-5-nano
+
+# or run a single GAIA Level-1 question (no REPL)
+python -m banna_agent.benchmarks.gaia.runner \
+    --policy verifier_retry --provider openai --model gpt-5-nano \
+    --level 1 --n 1
+```
+
+### Example REPL session
+
+```
+$ banna --policy verifier_retry --provider openai --model gpt-5-nano
+
+● banna · v0.1.0   provider=openai   model=gpt-5-nano   policy=verifier_retry
+
+> How many studio albums did Mercedes Sosa release between 2000 and 2009?
+
+  thinking…
+  ▸ search(query="Mercedes Sosa discography studio albums 2000-2009")
+    ↳ 8 results · evidence_id ev_a3f
+  ▸ read_url(url="https://en.wikipedia.org/wiki/Mercedes_Sosa")
+    ↳ 12.4 kB · evidence_id ev_91c
+  thinking…
+  ▸ final_answer(answer="3", evidence_ids=["ev_a3f", "ev_91c"])
+  verifiers: format ✓  citation ✓  coverage ✓  arithmetic skip
+
+● banna
+  3
+
+  3 steps · 4.7s · 1840→210 tok · $0.0021
+
+> /show trace
+  …step-by-step dump of action + observation + meta…
+
+> /exit
+```
+
+The full GAIA validation runner (165 questions across L1/L2/L3) is in `experiments/02_gaia_full/run.py`.
+
+## Failure modes & fixes (the C1–C6 pass)
+
+Diagnosed from a full GAIA validation run on `gpt-5-nano`. Each fix lands as a structural change to the loop, with unit tests pinning the new behavior.
+
+| ID | Failure mode | Root cause | Fix |
+|----|--------------|------------|-----|
+| C1 | `[empty_reply]` loops eat the step budget | Repair-style THINKs counted as productive steps | New `Budget.repair_steps_used` axis + `max_repair_steps=6`; `meta["repair"]=True` routes off the main counter |
+| C2 | Model returns empty content + no tool call | No detection / no escape | After 2 consecutive empties with no evidence, force `tool_choice` to any tool (provider-agnostic) |
+| C3 | `pred_answer=null` on budget exhaustion | Loop exits with no commit | `policy.synthesize_on_exhaustion(state)`: one threaded LLM call with forced `final_answer` + cheap fallback chain |
+| C4 | L1 step cap too tight (8 steps) | Default budget profile | L1/L2/L3 caps bumped to 12/18/24 |
+| C5 | Rich file tools never used on attachments | Hint steered model toward `read_file` even for PDF/XLSX | Extension-routed `_file_hint()` + cheap `_file_summary()` (pypdf page count, openpyxl sheet names, CSV header) |
+| C6 | Verifier retries low repair rate | Feedback was generic | Each verifier populates `meta["nudge"]` with a verifier-specific actionable instruction; retry feedback groups by verifier |
+
+## GAIA validation results
+
+> The fixes are landed and tested (562 tests passing in this public repo, 568 in the private superset). The post-fix re-run on GAIA validation is the next thing on the queue.
+
+| Run | Provider · model | Policy | Set | Accuracy | Notes |
+|-----|------------------|--------|-----|----------|-------|
+| Pre-C1–C6 | OpenAI · `gpt-5-nano` | verifier_retry | GAIA val (165 Q) | **33.9 %** (56 / 165) | $0.94, 7 structural bugs surfaced |
+| Post-C1–C6 | OpenAI · `gpt-5-nano` | verifier_retry | GAIA val (165 Q) | _re-run pending_ | target ≥ 40 % |
+| Post-C1–C6 | OpenAI · `gpt-5-nano` | best_of_n (K=3) | GAIA val (165 Q) | _re-run pending_ | stretch ≥ 45 % |
+
+(Numbers populate once the re-run completes; CI is wired to gate on a held-out smoke subset.)
+
+## Repo layout
+
+```
+src/banna_agent/
+├── core/          AgentState, Trace, Action, Budget, EventLog, run_policy
+├── llm/           provider-agnostic LLMClient + adapters (anthropic, openai, gemini, ollama, bedrock)
+├── tools/         search, read_url, read_file, pdf/xlsx tools, python_sandbox,
+│                  calculator, run_shell, grep, list_files, plan, memory, final_answer
+├── policies/      react, planner_react, verifier_retry, bfs/dfs/best_first_over_plans, best_of_n
+├── verifiers/     arithmetic, citation, coverage, format, command (+ base protocol)
+├── benchmarks/    gaia/ (loader, runner, scorer, report)
+├── memory/        in_memory_store, jsonl_store, skill_library, embeddings
+└── cli/           Rich-based REPL: /policy /budget /show /skills /compact /save /load …
+```
+
+Tests live in `tests/` and are organized to mirror `src/`. Run them with:
+
+```bash
+pytest -q
+```
+
+Current status: **562 passed, 0 failed** on this public branch (no external substrate dependencies).
+
+## License
+
+MIT

banna-0.1.0/pyproject.toml

@@ -0,0 +1,80 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "banna"
+version = "0.1.0"
+description = "A from-scratch, provider-agnostic reasoning agent with a typed state substrate and verifier-guided search. Primary benchmark: GAIA."
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{ name = "Siavash Monfared", email = "monfared@alum.mit.edu" }]
+license = { text = "MIT" }
+keywords = ["agent", "llm", "react", "verifier", "gaia", "reasoning", "anthropic", "openai", "gemini"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: MacOS",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+dependencies = [
+    "anthropic>=0.40",
+    "openai>=1.50",
+    "google-generativeai>=0.8",
+    "requests>=2.31",
+    "pydantic>=2.6",
+    "datasets>=2.19",
+    "pyyaml>=6.0",
+    "beautifulsoup4>=4.12",
+    "pypdf>=4.0",
+    "openpyxl>=3.1",
+    "pandas>=2.2",
+    "pillow>=10.0",
+    "rich>=13.7",
+]
+
+[project.urls]
+Homepage = "https://github.com/siavashmonfared/banna"
+Repository = "https://github.com/siavashmonfared/banna"
+Issues = "https://github.com/siavashmonfared/banna/issues"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-cov>=5.0",
+    "ruff>=0.5",
+    "mypy>=1.10",
+]
+# Optional PDF-tables support. pypdf (core dep) handles text extraction;
+# pdfplumber adds table-structure parsing for GAIA tasks where the
+# answer lives in a PDF table. Install with `pip install ".[pdf]"`.
+pdf = [
+    "pdfplumber>=0.11",
+]
+
+[project.scripts]
+banna = "banna_agent.cli:main"
+banna-agent = "banna_agent.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "B", "SIM", "TID", "RUF"]
+ignore = ["E501"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-ra --strict-markers"

banna-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+