metaensemble 0.2.0__py3-none-any.whl

evals/configs/default.yaml

@@ -0,0 +1,59 @@
+# MetaEnsemble evaluation default config.
+#
+# Read by `metaensemble eval` to set per-cycle parameters. Override on
+# the command line with `--seeds`, `--budget-usd`, etc.
+
+cycle:
+  # Number of seeds per cell. Smoke = 1, full = 5, very thorough = 10.
+  seeds: 5
+  # Per-run USD budget passed to `claude --max-budget-usd`. A run that
+  # exceeds the budget counts as a failure for `pass@budget`.
+  budget_usd: 0.30
+  # Default models for each role in a cell. Cheap models handle
+  # parsing/pre-flight; the executor uses the manifest tier;
+  # the reviewer and synthesizer go higher only when needed.
+  model_routing:
+    parser: claude-haiku-4-5
+    executor: claude-sonnet-4-6
+    reviewer: claude-sonnet-4-6
+    synthesizer: claude-opus-4-7
+
+cells:
+  # Three baselines plus the canonical full system.
+  - id: B1_single_agent
+    kind: baseline
+    description: One model call, no Coordinator protocol, no Manifest, no Ledger.
+  - id: B2_single_agent_prompted
+    kind: baseline
+    description: B1 plus a strong system prompt naming role + acceptance criteria.
+  - id: B3_subagent_default
+    kind: baseline
+    description: Runtime's default subagent path; no MetaEnsemble layer.
+  - id: B4_best_prompt
+    kind: baseline
+    description: Best-effort single-agent baseline with same Manifest pointers
+      and acceptance criteria as MM. The strongest competitor MM has to beat.
+  - id: MM_full
+    kind: full_system
+    description: Coordinator + Manifest + Ledger + Quality gate.
+  # Three ablations.
+  - id: MM_minus_manifest
+    kind: ablation
+    description: Coordinator dispatches without composing a typed contract.
+  - id: MM_minus_ledger
+    kind: ablation
+    description: Coordinator dispatches without recording Run rows.
+    # Harness logs externally so removing the Ledger does not break measurement.
+  - id: MM_minus_quality_gate
+    kind: ablation
+    description: Coordinator dispatches without post-Deliverable quality check.
+
+reporting:
+  # Confidence interval method for pass-rates.
+  confidence: wilson_95
+  # Token-waste threshold (D-9). Failed-run waste above this fraction of
+  # total tokens blocks a "ship" verdict.
+  failed_run_waste_threshold: 0.10
+  # Orchestration-overhead ceiling (D-8). MM tokens / B4 tokens above
+  # this ratio blocks a "ship" verdict.
+  overhead_ratio_ceiling: 2.0

evals/datasets/suite_a/tasks.yaml

@@ -0,0 +1,123 @@
+# Suite A — eight software-engineering tasks.
+#
+# Each task gates the cell's pass-rate against measurable acceptance
+# criteria. The starting state is a frozen Git SHA so re-runs are
+# reproducible; the acceptance criteria are executed by `runners/api.py`
+# after the cell's deliverable lands.
+
+tasks:
+  - id: a1_bugfix_off_by_one
+    title: Fix an off-by-one in a paginator
+    description: |
+      The paginator at `pagination.py:42` returns one fewer item than
+      expected on every page boundary. Locate and fix the bug; add a
+      regression test.
+    starting_repo: oss-fixture-paginator
+    starting_sha: __DEFERRED__
+    acceptance:
+      - kind: build_passes
+      - kind: test_count_at_least
+        value: 5
+      - kind: lint_clean
+      - kind: file_modified
+        path: pagination.py
+      - kind: file_modified
+        path: test_pagination.py
+
+  - id: a2_refactor_module
+    title: Split a 600-line module into three cohesive files
+    description: |
+      The module `legacy/big_module.py` (612 lines) has three responsibilities.
+      Split it into three files preserving the public API. No behavior change.
+    starting_repo: oss-fixture-legacy
+    starting_sha: __DEFERRED__
+    acceptance:
+      - kind: build_passes
+      - kind: test_count_at_least
+        value: 12
+      - kind: api_surface_preserved
+
+  - id: a3_doc_update
+    title: Document split rollback commands in USER-GUIDE
+    description: |
+      The CLI has separate project and user rollback commands:
+      `metaensemble unadopt --purge-state` and
+      `metaensemble user-teardown --purge-state`. Add a paragraph to
+      USER-GUIDE.md under the rollback section.
+    starting_repo: metaensemble
+    starting_sha: __DEFERRED__
+    acceptance:
+      - kind: markdown_links_resolve
+      - kind: file_modified
+        path: USER-GUIDE.md
+
+  - id: a4_test_addition
+    title: Add tests for the reconcile module
+    description: |
+      The `metaensemble.lib.reconcile` module has 8 tests covering Layer-1 and
+      Layer-2 reconcile. Add a test that asserts a sidecar's
+      `manifest_id` survives reconciliation into the Run row.
+    starting_repo: metaensemble
+    starting_sha: __DEFERRED__
+    acceptance:
+      - kind: build_passes
+      - kind: test_count_delta_at_least
+        value: 1
+      - kind: file_modified
+        path: metaensemble/tests/test_reconcile.py
+
+  - id: a5_design_review
+    title: Review the uninstall-mode design
+    description: |
+      Read the "Uninstall modes" and "Recovery and rollback" sections of
+      DEPLOYMENT.md and produce a one-page review
+      (`reports/<date>-uninstall-review.md`) naming at least three risks
+      the documented design does not address.
+    starting_repo: metaensemble
+    starting_sha: __DEFERRED__
+    acceptance:
+      - kind: file_exists
+        glob: reports/*-uninstall-review.md
+      - kind: word_count_at_least
+        value: 300
+
+  - id: a6_security_review
+    title: Security-review the new transcript walker
+    description: |
+      The new `metaensemble/lib/transcript.py` reads JSONL from `transcript_path`.
+      Write `reports/<date>-transcript-security.md` listing every defensive
+      assumption and one concrete attack the walker survives.
+    starting_repo: metaensemble
+    starting_sha: __DEFERRED__
+    acceptance:
+      - kind: file_exists
+        glob: reports/*-transcript-security.md
+
+  - id: a7_perf_tune
+    title: Tune `get_window_burn` to <5ms p95 on 100k Runs
+    description: |
+      Current p95 is 9ms on 100k Runs. Add the missing index and assert the
+      new p95 in `metaensemble/tests/test_perf_ledger.py`.
+    starting_repo: metaensemble
+    starting_sha: __DEFERRED__
+    acceptance:
+      - kind: build_passes
+      - kind: perf_benchmark_passes
+        benchmark: test_get_window_burn_meets_p95
+
+  - id: a8_infra_change
+    title: Add the no-quality CI matrix axis
+    description: |
+      The CI workflow should run pytest a second time with the quality
+      runners absent. Update `.github/workflows/ci.yml` and
+      add `@pytest.mark.requires_radon` / `requires_bandit` markers to
+      the tests that need them.
+    starting_repo: metaensemble
+    starting_sha: __DEFERRED__
+    acceptance:
+      - kind: ci_yaml_has_matrix_axis
+        axis: no-quality
+
+# Per W8 sign-off, every starting_sha resolves to a real commit before
+# the harness ships its first benchmark. The __DEFERRED__ values are
+# placeholders so the test-time loader does not need a network round-trip.

evals/datasets/suite_b/items.yaml

@@ -0,0 +1,90 @@
+# Suite B — domain-specific classification smoke fixture.
+#
+# STATUS: SMOKE SUITE.
+#
+# Twelve labeled items is not enough to make calibration claims. This
+# file is the smoke suite: it proves the
+# pipeline end-to-end and lets a regression run in CI without
+# real-money API spend. It is NOT the calibration set.
+#
+# This fixture's current labels happen to use a narrow Somali dialect
+# classification task. That is an eval fixture, not MetaEnsemble's product
+# scope. MetaEnsemble is project-agnostic.
+#
+# Smoke items are written by a non-native Somali reader using
+# published dialect references. They are deliberately easy — clearly
+# Northern/Standard Somali — so a pass means the pipeline works,
+# not that the model is calibrated.
+
+items:
+  - id: b_smoke_01
+    text: "Maxaa waaye sababta aad maanta ii soo wici weyday, ma mashquul baad ahayd?"
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+    notes: |
+      Clear Northern Somali markers: "wici weyday" (negative habitual)
+      and the interrogative "ma … baad ahayd" pattern.
+
+  - id: b_smoke_02
+    text: "Meeqo qof baa shirka yimid, yaase ka hadlay arrintaas?"
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+    notes: |
+      "Baa" focus marker + "yimid" verb form pin the dialect to
+      Northern/Standard.
+
+  - id: b_smoke_03
+    text: "Sidee laga yaabaa in arrintaan loo xalliyo si toosan?"
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+    notes: Standard register; common in news/formal speech.
+
+  - id: b_smoke_04
+    text: "Markii uu yimid, waxaan u sheegay inay tahay arrin muhiim ah."
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+
+  - id: b_smoke_05
+    text: "Waxaan tagay suuqa subaxnimo, badda agteeda."
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+
+  - id: b_smoke_06
+    text: "Maxaad u maleyneysaa inay ka jirto fursad cusub?"
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+
+  - id: b_smoke_07
+    text: "Sannadkaan dugsiga ardayda waa la kordhay."
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+
+  - id: b_smoke_08
+    text: "Markaa sidee ayaad mooddaa arrintaas?"
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+
+  - id: b_smoke_09
+    text: "Wax ka qabasho ah waa la sameeyay, balse natiijada lama hubo."
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+
+  - id: b_smoke_10
+    text: "Haddii aanan tegin, ma waxaa lagaa filan inaad ka soo qaybgasho?"
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+
+  - id: b_smoke_11
+    text: "Cuntada caawa waa fiicnaa, mahad waxaad leedahay."
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+
+  - id: b_smoke_12
+    text: "Khayraadka dalka ku jira waxay u baahan yihiin maamul fiican."
+    gold_label: northern_standard
+    acceptable_labels: [northern_standard]
+
+# Acceptable labels left as a singleton set on every item is part of the
+# smoke-only contract. The calibration set will carry richer alternative
+# labels (Maay, Benadiri, Northern, mixed, code-switching) per the
+# failure-mode catalog the system card promises.

evals/runners/__init__.py

@@ -0,0 +1,12 @@
+"""MetaEnsemble evaluation runners.
+
+Three tiers:
+- replay: cassette-based, no API spend. PR gate.
+- smoke:  one seed × classification smoke set. Nightly.
+- full:   N seeds × every cell × every suite. Release gate.
+
+Modules:
+- api: tiered runner dispatch.
+- metrics: Wilson CI, pass@budget, quality_per_1k_tokens, overhead ratio.
+- replay: cassette reader for the PR tier.
+"""