bantamkit 0.27.0__py3-none-any.whl

bantamkit/assets/evals/tasks/extract-invoice.yaml

@@ -0,0 +1,14 @@
+name: extract-invoice
+family: structured-extraction
+prompt: |
+  Extract the invoice as JSON with keys "number" (string) and "total" (integer).
+  Text: "Invoice INV-42 came to 199 USD, paid by card."
+schema:
+  type: object
+  required: [number, total]
+  properties:
+    number: {type: string}
+    total: {type: integer}
+scoring:
+  kind: json_equal
+  expected: {number: "INV-42", total: 199}

bantamkit/assets/evals/tasks/extract-order.yaml

@@ -0,0 +1,15 @@
+name: extract-order
+family: structured-extraction
+prompt: |
+  Extract the order as JSON with keys "item" (singular, lowercase) and
+  "quantity" (integer).
+  Text: "Customer wants three Widgets shipped by Friday."
+schema:
+  type: object
+  required: [item, quantity]
+  properties:
+    item: {type: string}
+    quantity: {type: integer}
+scoring:
+  kind: json_equal
+  expected: {item: "widget", quantity: 3}

bantamkit/assets/evals/tasks/extract-schedule.yaml

@@ -0,0 +1,14 @@
+name: extract-schedule
+family: structured-extraction
+prompt: |
+  Extract the schedule as JSON with keys "day" (lowercase) and "time" (HH:MM).
+  Text: "Standup happens every Tuesday at 09:30 sharp."
+schema:
+  type: object
+  required: [day, time]
+  properties:
+    day: {type: string}
+    time: {type: string}
+scoring:
+  kind: json_equal
+  expected: {day: "tuesday", time: "09:30"}

bantamkit/assets/evals/tasks/extract-versions.yaml

@@ -0,0 +1,17 @@
+name: extract-versions
+family: structured-extraction
+prompt: |
+  Extract as JSON with keys "package" (string) and "versions" (array of strings,
+  in the order mentioned).
+  Text: "Package foo supports versions 1.2, 1.3 and 2.0."
+schema:
+  type: object
+  required: [package, versions]
+  properties:
+    package: {type: string}
+    versions:
+      type: array
+      items: {type: string}
+scoring:
+  kind: json_equal
+  expected: {package: "foo", versions: ["1.2", "1.3", "2.0"]}

bantamkit/assets/evals/tasks/nav-prod-port.yaml

@@ -0,0 +1,84 @@
+name: nav-prod-port
+family: file-nav
+tools:
+- read_file
+- list_files
+workspace:
+  README.md: '# billing-svc
+
+
+    Billing service. Deployment details live in docs/deploy.md.
+
+    Development setup is in docs/dev.md (local only).
+
+
+    For coding conventions see style.md.'
+  docs/deploy.md: '# Deploying billing-svc
+
+
+    Production settings are NOT here. Every environment value
+
+    (host, port, pool sizes) is read from config/prod.yaml at boot.
+
+    Staging mirrors production except where config/staging.yaml overrides.'
+  config/prod.yaml: "host: billing.internal.example.com\nport: 9443\npool:\n  min: 4\n  max: 32\ntls: required\n"
+  config/staging.yaml: "host: billing.staging.example.com\nport: 8443\npool:\n  min: 1\n  max: 8\ntls: optional\n"
+  style.md: '# Style guide (excerpt)
+
+
+    Prefer small modules with one clear responsibility. Public functions carry
+
+    docstrings describing behavior, not implementation. Errors are values where
+
+    practical; exceptions for programmer mistakes. Log lines are structured
+
+    key=value pairs, lowercase keys. Feature flags default off and are removed
+
+    within two releases of full rollout. Database migrations are forward-only;
+
+    rollbacks happen by writing a new migration. Review comments prefixed nit:
+
+    are optional. Vendored code lives under third_party/ and is never edited in
+
+    place — patch files only. Commit subjects use the imperative mood and stay
+
+    under 68 characters. Squash-merge is the default; merge commits are for
+
+    release branches only. Generated files carry a do-not-edit header and are
+
+    regenerated in CI, never by hand.
+
+    '
+  ci.md: '# CI pipeline notes
+
+
+    The pipeline runs on every push: lint, unit tests, integration tests, and a
+
+    packaging step. Caching of the dependency layer cut average build time from
+
+    11 minutes to 6. Flaky tests are quarantined under tests/flaky/ and re-run
+
+    three times before being reported as failures. The nightly job additionally
+
+    runs the slow property-based suite and publishes coverage to the internal
+
+    dashboard. Artifacts older than 30 days are pruned automatically. If a
+
+    release tag is pushed, the pipeline also builds the signed bundle and
+
+    uploads it to the artifact store with a 90-day retention override. Secrets
+
+    are injected via the runner''s OIDC exchange; never commit tokens. When the
+
+    integration environment is unavailable the pipeline degrades to unit tests
+
+    only and marks the run amber, not red. Amber runs do not block merges but
+
+    three consecutive ambers page the on-call.
+
+    '
+prompt: 'Find the production port of billing-svc from the workspace files. Start with list_files, then follow the documentation to the right file. Use list_files first, then read_file on every file you need — follow each pointer the files give you; do not answer from memory or guess. Answer with ONLY this JSON, nothing else: {"port": <number>}'
+scoring:
+  kind: json_equal
+  expected:
+    port: 9443

bantamkit/assets/evals/tasks/nav-release-bundle.yaml

@@ -0,0 +1,87 @@
+name: nav-release-bundle
+family: file-nav
+tools:
+- read_file
+- list_files
+workspace:
+  README.md: '# imgproc
+
+
+    Image processing CLI. Release process: docs/release.md.'
+  docs/release.md: '# Releasing imgproc
+
+
+    The bundle name is assembled as <name>-<version>.tar.gz where <name>
+
+    comes from package.cfg and <version> comes from VERSION. Both files
+
+    live at the repo root. Never hardcode either value.'
+  package.cfg: '[package]
+
+    name = imgproc-cli
+
+    license = apache-2.0
+
+    '
+  VERSION: '2.9.1
+
+    '
+  docs/ci.md: '# CI pipeline notes
+
+
+    The pipeline runs on every push: lint, unit tests, integration tests, and a
+
+    packaging step. Caching of the dependency layer cut average build time from
+
+    11 minutes to 6. Flaky tests are quarantined under tests/flaky/ and re-run
+
+    three times before being reported as failures. The nightly job additionally
+
+    runs the slow property-based suite and publishes coverage to the internal
+
+    dashboard. Artifacts older than 30 days are pruned automatically. If a
+
+    release tag is pushed, the pipeline also builds the signed bundle and
+
+    uploads it to the artifact store with a 90-day retention override. Secrets
+
+    are injected via the runner''s OIDC exchange; never commit tokens. When the
+
+    integration environment is unavailable the pipeline degrades to unit tests
+
+    only and marks the run amber, not red. Amber runs do not block merges but
+
+    three consecutive ambers page the on-call.
+
+    '
+  docs/style.md: '# Style guide (excerpt)
+
+
+    Prefer small modules with one clear responsibility. Public functions carry
+
+    docstrings describing behavior, not implementation. Errors are values where
+
+    practical; exceptions for programmer mistakes. Log lines are structured
+
+    key=value pairs, lowercase keys. Feature flags default off and are removed
+
+    within two releases of full rollout. Database migrations are forward-only;
+
+    rollbacks happen by writing a new migration. Review comments prefixed nit:
+
+    are optional. Vendored code lives under third_party/ and is never edited in
+
+    place — patch files only. Commit subjects use the imperative mood and stay
+
+    under 68 characters. Squash-merge is the default; merge commits are for
+
+    release branches only. Generated files carry a do-not-edit header and are
+
+    regenerated in CI, never by hand.
+
+    '
+prompt: 'What is the exact release bundle filename for the current version? Follow the release documentation and assemble it from the files it names. Use list_files first, then read_file on every file you need — follow each pointer the files give you; do not answer from memory or guess. Answer with ONLY this JSON, nothing else: {"bundle": "<filename>"}'
+scoring:
+  kind: json_equal
+  expected:
+    bundle: imgproc-cli-2.9.1.tar.gz

bantamkit/assets/evals/tasks/recall-audit-retention.yaml

@@ -0,0 +1,17 @@
+name: recall-audit-retention
+family: memory-recall
+prompt: 'How many days do we keep AUDIT logs? If you have a memory tool, check memory
+  first. Answer with ONLY this JSON, nothing else: {"days": <number>}'
+memory_setup:
+- type: project
+  name: log-retention-default
+  description: how long ordinary application logs are kept
+  body: Application logs are kept for 45 days.
+- type: project
+  name: log-retention-audit
+  description: retention exception for the audit trail
+  body: Audit logs are the exception and are kept for 400 days.
+scoring:
+  kind: json_equal
+  expected:
+    days: 400

bantamkit/assets/evals/tasks/recall-cache-ttl.yaml

@@ -0,0 +1,13 @@
+name: recall-cache-ttl
+family: memory-recall
+prompt: 'What is the cache TTL for the pricing service? If you have a memory tool,
+  check memory first. Answer with ONLY this JSON, nothing else: {"seconds": <number>}'
+memory_setup:
+- type: project
+  name: pricing-cache-ttl
+  description: cache ttl seconds for the pricing service
+  body: The pricing service caches responses for 240 seconds.
+scoring:
+  kind: json_equal
+  expected:
+    seconds: 240

bantamkit/assets/evals/tasks/recall-db-port.yaml

@@ -0,0 +1,17 @@
+name: recall-db-port
+family: memory-recall
+prompt: 'What port does the STAGING database listen on? If you have a memory tool,
+  check memory first. Answer with ONLY this JSON, nothing else: {"port": <number>}'
+memory_setup:
+- type: project
+  name: staging-db-port
+  description: staging database port number
+  body: The staging database listens on port 5433.
+- type: project
+  name: prod-db-connection
+  description: production postgres connection endpoint
+  body: Production postgres is at db.prod.internal on port 5432.
+scoring:
+  kind: json_equal
+  expected:
+    port: 5433

bantamkit/assets/evals/tasks/recall-deploy.yaml

@@ -0,0 +1,13 @@
+name: recall-deploy
+family: memory-recall
+prompt: 'How do we deploy this project to production? If you have a memory tool,
+  check memory first. Answer with ONLY this JSON, nothing else: {"command": "<command>"}'
+memory_setup:
+- type: project
+  name: deploy-command
+  description: how we deploy this project to production
+  body: Deploy with `make ship-prod` from the repo root.
+scoring:
+  kind: json_equal
+  expected:
+    command: make ship-prod

bantamkit/assets/evals/tasks/recall-env-endpoint.yaml

@@ -0,0 +1,18 @@
+name: recall-env-endpoint
+family: memory-recall
+prompt: 'What is the full base URL for the reports API in PRODUCTION, including
+  the version prefix? If you have a memory tool, check memory first. Answer with
+  ONLY this JSON, nothing else: {"url": "<url>"}'
+memory_setup:
+- type: project
+  name: prod-api-host
+  description: hostname serving production traffic
+  body: Production traffic is served from https://api.example-prod.io.
+- type: project
+  name: reports-version-prefix
+  description: version path segment used by the reports service
+  body: The reports service is mounted under /v3/reports on every host.
+scoring:
+  kind: json_equal
+  expected:
+    url: https://api.example-prod.io/v3/reports

bantamkit/assets/evals/tasks/recall-oncall-rotation.yaml

@@ -0,0 +1,21 @@
+name: recall-oncall-rotation
+family: memory-recall
+prompt: 'Who is on call for the PAYMENTS service this week? If you have a memory
+  tool, check memory first. Answer with ONLY this JSON, nothing else: {"name": "<name>"}'
+memory_setup:
+- type: project
+  name: oncall-payments
+  description: current pager duty for the payments service
+  body: Payments on-call this week is Priya.
+- type: project
+  name: oncall-search
+  description: rotation owner covering search infrastructure
+  body: Search on-call this week is Marcus.
+- type: project
+  name: oncall-ingest
+  description: escalation contact for the ingest pipeline
+  body: Ingest on-call this week is Dana.
+scoring:
+  kind: json_equal
+  expected:
+    name: Priya

bantamkit/assets/evals/tasks/recall-oncall.yaml

@@ -0,0 +1,13 @@
+name: recall-oncall
+family: memory-recall
+prompt: 'Who is on-call for infrastructure this quarter? If you have a memory tool,
+  check memory first. Answer with ONLY this JSON, nothing else: {"name": "<name>"}'
+memory_setup:
+- type: project
+  name: infra-oncall
+  description: who is on-call for infrastructure this quarter
+  body: Nadia is on-call for infrastructure until end of Q3.
+scoring:
+  kind: json_equal
+  expected:
+    name: Nadia

bantamkit/assets/evals/tasks/recall-org-quota.yaml

@@ -0,0 +1,18 @@
+name: recall-org-quota
+family: memory-recall
+prompt: 'What is the TOTAL requests-per-minute quota for one full org on the api
+  gateway? If you have a memory tool, check memory first, then compute the answer.
+  Answer with ONLY this JSON, nothing else: {"total": <number>}'
+memory_setup:
+- type: project
+  name: gateway-user-quota
+  description: per-user rate limit on the api gateway
+  body: The api gateway allows each user 40 requests per minute.
+- type: project
+  name: org-seat-count
+  description: how many seats one org licence includes
+  body: Every org licence includes exactly 5 user seats.
+scoring:
+  kind: json_equal
+  expected:
+    total: 200

bantamkit/assets/evals/tasks/recall-owner.yaml

@@ -0,0 +1,13 @@
+name: recall-owner
+family: memory-recall
+prompt: 'Which team owns the payments API? If you have a memory tool, check memory
+  first. Answer with ONLY this JSON, nothing else: {"team": "<team name>"}'
+memory_setup:
+- type: project
+  name: payments-api-owner
+  description: which team owns the payments api
+  body: The payments API is owned by team Atlas.
+scoring:
+  kind: json_equal
+  expected:
+    team: Atlas

bantamkit/assets/evals/tasks/shop-basket-total.yaml

@@ -0,0 +1,10 @@
+name: shop-basket-total
+family: tool-use
+prompt: |
+  A customer orders 2 widgets, 3 doohickeys and 1 gadget. Use the tools to
+  look up unit prices, then answer with ONLY this JSON, nothing else:
+  {"total": <number>}
+tools: [price_lookup]
+scoring:
+  kind: json_equal
+  expected: {total: 131}

bantamkit/assets/evals/tasks/shop-cheapest.yaml

@@ -0,0 +1,9 @@
+name: shop-cheapest
+family: tool-use
+prompt: |
+  Use the tools to check the unit prices of "widget" and "gadget".
+  Answer with ONLY this JSON, nothing else: {"cheaper": "<item name>"}
+tools: [price_lookup]
+scoring:
+  kind: json_equal
+  expected: {cheaper: "widget"}

bantamkit/assets/evals/tasks/shop-compare.yaml

@@ -0,0 +1,9 @@
+name: shop-compare
+family: tool-use
+prompt: |
+  Use the tools to check the unit prices of "widget" and "gadget",
+  and answer with the name of the more expensive item.
+tools: [price_lookup]
+scoring:
+  kind: tool_trace
+  expected: [price_lookup, price_lookup]

bantamkit/assets/evals/tasks/shop-gadget-value.yaml

@@ -0,0 +1,9 @@
+name: shop-gadget-value
+family: tool-use
+prompt: |
+  Use the tools to find the unit price and stock count of "gadget",
+  then answer with the total value of the stock (price times stock) as a number.
+tools: [price_lookup, stock_lookup]
+scoring:
+  kind: contains
+  expected: ["540"]

bantamkit/assets/evals/tasks/shop-stock-total.yaml

@@ -0,0 +1,9 @@
+name: shop-stock-total
+family: tool-use
+prompt: |
+  Use the tools to find the stock counts of "widget" and "gadget",
+  then answer with the combined total stock as a number.
+tools: [stock_lookup]
+scoring:
+  kind: contains
+  expected: ["13"]

bantamkit/assets/evals/tasks/shop-total.yaml

@@ -0,0 +1,9 @@
+name: shop-total
+family: tool-use
+prompt: |
+  Use the tools to find the unit price and stock count of "widget",
+  then answer with the total value of the stock (price times stock) as a number.
+tools: [price_lookup, stock_lookup]
+scoring:
+  kind: contains
+  expected: ["100"]

bantamkit/assets/profiles/default.yaml

@@ -0,0 +1,31 @@
+# The conservative default profile. Every number here was calibrated on
+# qwen3:4b-instruct (the reference model) — the cross-model sweep measured
+# that honestly; per-model profiles are future work (P6).
+name: default
+agent:
+  max_turns: 10
+  observation_budget: 4096
+structured:
+  max_retries: 3
+schema_gate:
+  max_attempts: 3
+# One retry only: the gate rescues a missing-JSON restatement, and a second
+# miss is format abandonment that more nagging does not fix (spec §2.4).
+json_answer:
+  max_attempts: 1
+critique:
+  max_rounds: 3
+  evidence_budget: 4096
+# Global spend governor (P3), applied only when a TokenBudget is attached.
+# `ceiling` is the hard stop for one run; past `ceiling * optional_cutoff`
+# optional work (critique rounds) is denied. The gap between the two IS the
+# reserve that keeps the final answer emission affordable.
+token_budget:
+  ceiling: 6000
+  optional_cutoff: 0.75
+# Loop-detection thresholds, applied only when a LoopGuard is attached. Probe-
+# calibrated: max observation-repeat streak in every passing run was 2, looping
+# runs burned 6-16 — so the note fires at 3 and the hard warning at 5.
+loop_guard:
+  inject_at: 3
+  warn_at: 5

bantamkit/assets/profiles/patient.yaml

@@ -0,0 +1,31 @@
+# Default, but with a longer turn budget: for models that need more steps than
+# the 4b-calibrated default (measured: 3b file-nav and 7b recall died of turn
+# exhaustion under max_turns 10). Calibration-only until a bar says otherwise.
+name: patient
+agent:
+  max_turns: 16
+  observation_budget: 4096
+structured:
+  max_retries: 3
+schema_gate:
+  max_attempts: 3
+# One retry only: the gate rescues a missing-JSON restatement, and a second
+# miss is format abandonment that more nagging does not fix (spec §2.4).
+json_answer:
+  max_attempts: 1
+critique:
+  max_rounds: 3
+  evidence_budget: 4096
+# Global spend governor (P3), applied only when a TokenBudget is attached.
+# `ceiling` is the hard stop for one run; past `ceiling * optional_cutoff`
+# optional work (critique rounds) is denied. The gap between the two IS the
+# reserve that keeps the final answer emission affordable.
+token_budget:
+  ceiling: 6000
+  optional_cutoff: 0.75
+# Loop-detection thresholds, applied only when a LoopGuard is attached. Probe-
+# calibrated: max observation-repeat streak in every passing run was 2, looping
+# runs burned 6-16 — so the note fires at 3 and the hard warning at 5.
+loop_guard:
+  inject_at: 3
+  warn_at: 5

bantamkit/assets/rubrics/code-quality.yaml

@@ -0,0 +1,20 @@
+name: code-quality
+threshold: 7
+schema:
+  type: object
+  required: [score, feedback]
+  properties:
+    score: {type: integer, minimum: 0, maximum: 10}
+    feedback: {type: string}
+prompt: |
+  You are a strict code reviewer. Judge the code below.
+
+  Task:
+  {task}
+
+  Code:
+  {output}
+
+  Score 0-10 on: correctness for the task, handling of error cases, and
+  absence of dead or needless code. 10 = ship as-is.
+  Return ONLY JSON: {{"score": <int>, "feedback": "<specific defects to fix>"}}

bantamkit/assets/rubrics/grounded-completion.yaml

@@ -0,0 +1,37 @@
+name: grounded-completion
+threshold: 7
+schema:
+  type: object
+  required: [reasoning, score, feedback]
+  properties:
+    reasoning: {type: string}
+    score: {type: integer, minimum: 0, maximum: 10}
+    feedback: {type: string}
+prompt: |
+  You are a reviewer checking whether the answer contains the correct content.
+  The tool evidence below is the ground truth: it lists every tool call the
+  answerer made and what the tool returned. Verify the answer against it and
+  recompute any numbers yourself from the evidence. If the answer states a
+  fact or number that contradicts the evidence, score it 0-4 and put the
+  correct values from the evidence in your feedback.
+  In the reasoning field, first work out what the correct answer to the task
+  is, step by step, using ONLY the values in the evidence — do the arithmetic
+  and comparisons yourself. Then compare the given answer against your result.
+  Judge ONLY content. Do NOT deduct points for formatting, phrasing, extra
+  surrounding text, hedging, or verbosity. An answer that refuses or declines
+  to provide what the task asks for is missing the required content — score
+  it 0-4, even when the refusal is polite or explains itself.
+
+  Task:
+  {task}
+
+  Tool evidence:
+  {evidence}
+
+  Answer:
+  {output}
+
+  Score 0-10: 9-10 = required content present, correct, and consistent with
+  the evidence; 5-8 = partially correct or missing pieces; 0-4 = wrong,
+  absent, or contradicted by the evidence.
+  Return ONLY JSON: {{"reasoning": "<derive the correct answer from the evidence, then compare the given answer to it>", "score": <int>, "feedback": "<what is wrong or missing, and the correct values from the evidence>"}}

bantamkit/assets/rubrics/task-completion.yaml

@@ -0,0 +1,28 @@
+name: task-completion
+threshold: 7
+schema:
+  type: object
+  required: [score, feedback]
+  properties:
+    score: {type: integer, minimum: 0, maximum: 10}
+    feedback: {type: string}
+prompt: |
+  You are a reviewer checking whether the answer contains the correct content.
+  Judge ONLY whether the information the task asks for is present and correct.
+  Do NOT deduct points for formatting, phrasing, extra surrounding text,
+  hedging, or verbosity. If the required facts are present and right, the
+  answer completes the task.
+  An answer that refuses or declines to provide what the task asks for is
+  missing the required content — score it 0-4, even when the refusal is
+  polite or explains itself. Hedging around a real answer is fine; hedging
+  instead of an answer is not.
+
+  Task:
+  {task}
+
+  Answer:
+  {output}
+
+  Score 0-10: 9-10 = required content present and correct; 5-8 = partially
+  correct or missing pieces; 0-4 = wrong or absent.
+  Return ONLY JSON: {{"score": <int>, "feedback": "<what content is wrong or missing>"}}