tribes 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,11 @@
1
+ __pycache__/
2
+ *.pyc
3
+ .venv/
4
+ .pytest_cache/
5
+ dist/
6
+ build/
7
+ *.egg-info/
8
+ .env
9
+ .env.*
10
+ uv.lock
11
+ site/
tribes-1.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Daglar Dragomirov
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
tribes-1.1.0/PKG-INFO ADDED
@@ -0,0 +1,280 @@
1
+ Metadata-Version: 2.5
2
+ Name: tribes
3
+ Version: 1.1.0
4
+ Summary: Inference-time answer selection for optimized DSPy programs (GEPA, MIPROv2, SIMBA, ...).
5
+ Project-URL: Homepage, https://github.com/daglar-dragomirov/tribes
6
+ Project-URL: Repository, https://github.com/daglar-dragomirov/tribes
7
+ Project-URL: Issues, https://github.com/daglar-dragomirov/tribes/issues
8
+ Author: Daglar Dragomirov
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: agents,dspy,gepa,llm,prompt optimization,react,selection
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
20
+ Classifier: Typing :: Typed
21
+ Requires-Python: >=3.10
22
+ Requires-Dist: dspy>=3.0
23
+ Requires-Dist: litellm>=1.60
24
+ Provides-Extra: dev
25
+ Requires-Dist: pytest-cov>=5; extra == 'dev'
26
+ Requires-Dist: pytest>=8; extra == 'dev'
27
+ Requires-Dist: ruff>=0.6; extra == 'dev'
28
+ Provides-Extra: test
29
+ Requires-Dist: pytest-cov>=5; extra == 'test'
30
+ Requires-Dist: pytest>=8; extra == 'test'
31
+ Description-Content-Type: text/markdown
32
+
33
+ <p align="center">
34
+ <picture>
35
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/daglar-dragomirov/tribes/main/assets/lockup-dark.png">
36
+ <img src="https://raw.githubusercontent.com/daglar-dragomirov/tribes/main/assets/lockup-light.png" alt="TRIBES" width="460">
37
+ </picture>
38
+ </p>
39
+
40
+ <p align="center">
41
+ Inference-time answer selection for optimized DSPy programs
42
+ </p>
43
+
44
+ <p align="center">
45
+ <a href="https://daglar-dragomirov.github.io/tribes/"><b>Website</b></a> &nbsp;|&nbsp;
46
+ <a href="https://daglar-dragomirov.github.io/tribes/guides/quickstart/"><b>Quick Start</b></a> &nbsp;|&nbsp;
47
+ <a href="https://daglar-dragomirov.github.io/tribes/results/"><b>Results</b></a> &nbsp;|&nbsp;
48
+ <a href="research/"><b>Research</b></a> &nbsp;|&nbsp;
49
+ <a href="https://pypi.org/project/tribes/"><b>PyPI</b></a>
50
+ </p>
51
+
52
+ <p align="center">
53
+ <a href="https://pypi.org/project/tribes/"><img src="https://img.shields.io/pypi/v/tribes?logo=pypi&logoColor=white&label=pypi" alt="PyPI"></a>
54
+ <a href="https://pypi.org/project/tribes/"><img src="https://img.shields.io/pypi/pyversions/tribes?logo=python&logoColor=white" alt="Python"></a>
55
+ <a href="https://github.com/daglar-dragomirov/tribes/stargazers"><img src="https://img.shields.io/github/stars/daglar-dragomirov/tribes?logo=github&label=stars" alt="Stars"></a>
56
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License MIT"></a>
57
+ </p>
58
+
59
+ ---
60
+
61
+ ## What is TRIBES?
62
+
63
+ A prompt optimizer such as GEPA, MIPROv2 or SIMBA turns your data into one fixed [DSPy](https://github.com/stanfordnlp/dspy)
64
+ program. That program is good, but it is also brittle: it cannot look anything up, cannot run code to check itself, and
65
+ every answer is a single sample. TRIBES does not touch the optimization. It wraps the finished program with a
66
+ reviewer of its draft and an independent agent, picks one answer per input with a label-free rule, and switches itself
67
+ off on any task where it is not at least as good as the program.
68
+
69
+ ## Install
70
+
71
+ ```bash
72
+ pip install tribes
73
+ ```
74
+
75
+ ## Quickstart
76
+
77
+ ```python
78
+ import dspy
79
+ import tribes
80
+ from tribes import Tribes
81
+
82
+ dspy.configure(lm=dspy.LM("openrouter/z-ai/glm-5.3-flash"))
83
+
84
+ program = dspy.GEPA(metric=metric, ...).compile(student, trainset=train, valset=val) # or MIPROv2, SIMBA, ...
85
+
86
+ pt = Tribes(program, mode="full", examples=train,
87
+ task=tribes.tasks.math(), # optional: how answers compare, checks, tools
88
+ verifier=True) # optional: settle contested items by a check
89
+ pt.fit(train + val, metric) # decides once whether the layer (and the verifier) helps this task
90
+ print(pt.report["use_layer"], pt.report["mean_gain"])
91
+
92
+ prediction = pt(question="...") # same call signature as your program
93
+ ```
94
+
95
+ `fit` only needs labeled data the optimizer already saw; nothing is tuned per dataset.
96
+
97
+ ## How it works
98
+
99
+ ### Three forms
100
+
101
+ | form | what it is |
102
+ |---|---|
103
+ | **rigid** | the optimized program itself, unchanged |
104
+ | **assembled** | the program writes a draft; a ReAct reviewer checks it against the task and fixes it only if it finds a concrete defect |
105
+ | **liquid** | a ReAct agent that solves the task from scratch |
106
+
107
+ Both agents get the same task-agnostic instructions: the program's own instructions (the task's purpose plus what the
108
+ optimizer learned), a few correct outputs from your examples, and one fixed template per form. They have `run_python`
109
+ and any task tools you pass. There is no task-specific prompt text anywhere.
110
+
111
+ ### One answer per input
112
+
113
+ 1. Drop forms whose output is missing or unparsable.
114
+ 2. If you pass a deterministic, label-free `checker` (public unit tests, a SQL query that must run, a plan validator), drop
115
+ forms that fail it. If all fail, keep all.
116
+ 3. If two forms give the same answer, return it. Answers are compared with `answer_key` (default: normalized text).
117
+ 4. Otherwise return the reviewer's answer, then the program's, then the agent's.
118
+
119
+ In `full` mode the program and the reviewer run concurrently, and the agent only when they disagree. `lite` mode runs
120
+ the reviewer only. Every form and every retry is an independent request (its own cache entry), so the vote never
121
+ compares a form with a cached copy of itself.
122
+
123
+ ### Contested inputs: the verifier
124
+
125
+ When no two forms agree, an optional verifier settles the input by evidence, not opinion: it must design and run one
126
+ check whose outcome differs between the candidates (substituting an answer back, code on edge cases, an auxiliary SQL
127
+ query), and code, not the model, accepts a switch only if the quoted evidence matches a tool observation. A switch needs
128
+ two of three votes. `fit` enables the verifier per task only if it wins clearly more items than it loses.
129
+
130
+ ### Task adapters
131
+
132
+ `tribes.tasks` ships what the research used per task type, all label-free: `math()` (answers compare by value),
133
+ `choice()` (option letters), `sql(db_for)` (result-set agreement, a read-only `run_sql` tool, "the query runs"),
134
+ `code(tests_for)` (behavioural agreement on the public-test inputs, `run_public_tests`), `plan(validate)` (stated
135
+ constraints), `tool_qa()` (an abstention is not an answer) and `free_text()`.
136
+
137
+ For outputs that are sets of items scored by recall (retrieved documents, cited sources), `doc_set()` merges instead of
138
+ selecting: an element-wise vote over the forms' sets, capped at the program's own output size. On HoVer this beat
139
+ taking the reviewer's set by about +6 pp on development data for every optimizer.
140
+
141
+ ### Never worse than the program
142
+
143
+ `fit` runs the forms on labeled selection data and compares TRIBES with the program item by item. With the default
144
+ `switch="do_no_harm"` (the 1.0 rule) the layer is used unless it is shown worse by more than one clustered standard
145
+ error; `switch="non_inferiority"` accepts it only if it is shown to be at most `margin` worse. `switch_power` gives the
146
+ operating characteristic of either for your sample size. If the layer is not used, the wrapped object is exactly the
147
+ program, at exactly its cost.
148
+
149
+ ## Results
150
+
151
+ GLM 5.3 Flash as the task model; each optimizer run with its official settings; every number is TRIBES minus that
152
+ optimizer's own program on the same items.
153
+
154
+ **Fresh test (TEST-B, TRIBES 1.1)** — 1,010 new items, used nowhere in development or in the sealed test; the 1.1
155
+ policy and the analysis were pre-registered before any model saw them:
156
+
157
+ | benchmark (test items) | over GEPA | over MIPROv2 | over SIMBA | benchmark mean, 95 % CI |
158
+ |---|---|---|---|---|
159
+ | HoVer (150) | +12.7 | +8.0 | +16.7 | **+12.4** [+8.9, +16.4] |
160
+ | HotpotQA (200) | +7.0 | +3.0 | +2.5 | **+4.2** [+1.8, +6.8] |
161
+ | BIRD (300) | 0.0 | +5.0 | +4.7 | **+3.2** [+0.9, +5.7] |
162
+ | ToolHop (200) | +3.5 | +3.5 | −0.5 | +2.2 [−0.5, +5.0] |
163
+ | GPQA (160) | +0.6 | +4.4 | 0.0 | +1.7 [−1.5, +4.8] |
164
+ | **mean** | +4.8 | +4.8 | +4.7 | **+4.7** [+3.4, +6.1] |
165
+
166
+ Bold: significant (one-sided α = 0.025, Holm over the five benchmarks; the mean: p = 0.0001). Pre-registered
167
+ expectation: about +6 pp. On the same items TRIBES 1.0-lite gains +2.5 pp, so 1.1 adds **+2.2 pp** [+1.5, +3.0] over 1.0.
168
+ On HoVer the set merge beats merging three samples of the program by **+7.1 pp** [+4.4, +10.2]: the gain is not just
169
+ more sampling.
170
+
171
+ **Sealed test (TEST-A)** — opened once, after the rule and every per-task decision were frozen (TRIBES 1.0-lite):
172
+
173
+ | benchmark (test items) | over GEPA | over MIPROv2 | over SIMBA |
174
+ |---|---|---|---|
175
+ | NATURAL PLAN (200) | **+13.5** | **+17.0** | **+10.5** |
176
+ | LiveCodeBench (82) | **+11.0** | **+14.6** | **+13.4** |
177
+ | BIRD (200) | +2.5 | **+9.0** | +3.5 |
178
+ | GPQA (80) | +3.8 | 0.0 | +3.8 |
179
+ | ToolHop (200) | +2.5 | +2.0 | −0.5 (1 item of 200) |
180
+ | **mean** | **+6.7** | **+8.5** | **+6.1** |
181
+
182
+ Bold: 95 % confidence interval above zero. No cell has its interval below zero.
183
+
184
+ **Development** — 13 benchmarks (AIME, GPQA, LiveBench, LiveCodeBench, HotpotQA, HoVer, IFBench, PUPA, BFCL, τ²-bench,
185
+ NATURAL PLAN, BIRD, ToolHop) × 3 optimizers = 36 cells:
186
+
187
+ Per benchmark (TRIBES 1.0 full; bold: 95 % interval above zero; — : not run):
188
+
189
+ | benchmark | over GEPA | over MIPROv2 | over SIMBA |
190
+ |---|---|---|---|
191
+ | AIME | **+29.3** | **+23.3** | **+26.0** |
192
+ | NATURAL PLAN | **+24.0** | **+18.0** | **+21.0** |
193
+ | LiveCodeBench | **+17.0** | **+17.0** | **+18.0** |
194
+ | LiveBench | **+14.8** | **+19.3** | **+13.5** |
195
+ | BIRD | **+18.5** | +2.5 | **+6.5** |
196
+ | HoVer | **+6.7** | +3.7 | **+10.0** |
197
+ | GPQA | +5.1 | **+8.5** | +5.1 |
198
+ | ToolHop | **+8.0** | +0.5 | **+4.5** |
199
+ | HotpotQA | **+4.0** | +3.0 | **+5.7** |
200
+ | IFBench | **+2.2** | 0.0 | **+2.9** |
201
+ | PUPA | +1.7 | 0.0 | **+1.8** |
202
+ | τ²-bench | 0.0 | — | — |
203
+ | BFCL | 0.0 | — | 0.0 |
204
+
205
+ Averages:
206
+
207
+ | | full | lite |
208
+ |---|---|---|
209
+ | over GEPA | +10.1 pp | +9.3 pp |
210
+ | over MIPROv2 | +8.7 pp | +8.4 pp |
211
+ | over SIMBA | +9.6 pp | +8.9 pp |
212
+ | **all 36 cells** | **+9.5 pp** | **+8.9 pp** |
213
+
214
+ No cell below the program; 24 of 36 significantly above. The rule was designed after the development data was seen,
215
+ which is why the two test tables are the ones to quote.
216
+
217
+ **Research record**: how the rule was found, frozen and tested, with every number per cell, is in [`research/`](research/).
218
+
219
+ **Cost**: about 2.6× the program's inference cost in `lite` mode and 4.6× in `full` mode. There is no optimization
220
+ cost.
221
+
222
+ ## API
223
+
224
+ ```python
225
+ Tribes(program, *, mode="full", task=None, tools=(), answer_key=None, checker=None, examples=None,
226
+ verifier=None, switch="do_no_harm", z=None, margin=0.02, parallel=True, independent=True,
227
+ callbacks=(), prices=None, input_fields=None, output_field=None, max_iters=8, retries=3)
228
+ ```
229
+
230
+ | argument | meaning |
231
+ |---|---|
232
+ | `program` | any `dspy.Module`, usually an optimizer's output |
233
+ | `mode` | `"full"`: program + reviewer, agent on disagreement, vote · `"lite"`: reviewer only |
234
+ | `task` | a `tribes.tasks` adapter: answer key, checker, tools and verifier evidence for a task type |
235
+ | `tools` | task tools for both agents and the verifier (`run_python` is always included) |
236
+ | `answer_key` | `callable(prediction) -> hashable`: how two answers are compared |
237
+ | `checker` | `callable(inputs, prediction) -> bool \| None`: a deterministic, label-free check |
238
+ | `verifier` | `True` (or a `Verifier`) to settle contested inputs by an evidence-gated check |
239
+ | `switch`, `z`, `margin` | the task switch: `"do_no_harm"` (1.0) or `"non_inferiority"` |
240
+ | `parallel`, `independent` | run forms concurrently; give each form and retry its own cache entry |
241
+ | `callbacks`, `prices` | events (`LoggingCallback`, `JSONLCallback`, your own); `{model: (usd/M in, usd/M out)}` |
242
+
243
+ | method | what it does |
244
+ |---|---|
245
+ | `fit(selection_set, metric, num_threads=8, cluster=None)` | the task decision; `report` holds the evidence |
246
+ | `pt(**inputs)`, `await pt.acall(**inputs)` | one prediction |
247
+ | `explain(**inputs)` | a `Selection`: every form's output, keys, checks, the verifier's votes |
248
+ | `run_batch(items)`, `await abatch(items)` | many inputs, in order |
249
+ | `cost_report()` | calls, failures, seconds, tokens and dollars per form; cost relative to the program |
250
+ | `save_fit(path)`, `load_fit(path)` | persist the task decision |
251
+
252
+ Command line: `tribes replay rows.jsonl --mode both` applies the rules to saved per-item outputs with no model
253
+ calls; `tribes report fit.json` prints a saved decision.
254
+
255
+ ## FAQ
256
+
257
+ **Does it need a judge model?** No. Earlier versions used an LLM judge; it added +0.8 pp for most of the cost and was
258
+ removed. Agreement between independent forms and cheap deterministic checks do the job.
259
+
260
+ **Which optimizers?** Anything that produces a DSPy program. It was evaluated on GEPA, MIPROv2 and SIMBA.
261
+
262
+ **Is `run_python` a sandbox?** No. It runs code in a subprocess with a 10-second limit. Run untrusted workloads in an
263
+ isolated environment.
264
+
265
+ **What if it does not help my task?** Then `fit` says so (`use_layer = False`) and you get the program back, unchanged.
266
+
267
+ ## Citation
268
+
269
+ ```bibtex
270
+ @software{dragomirov2026tribes,
271
+ author = {Dragomirov, Daglar},
272
+ title = {TRIBES: Inference-time answer selection for optimized DSPy programs},
273
+ year = {2026},
274
+ url = {https://github.com/daglar-dragomirov/tribes}
275
+ }
276
+ ```
277
+
278
+ ## License
279
+
280
+ MIT
tribes-1.1.0/README.md ADDED
@@ -0,0 +1,248 @@
1
+ <p align="center">
2
+ <picture>
3
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/daglar-dragomirov/tribes/main/assets/lockup-dark.png">
4
+ <img src="https://raw.githubusercontent.com/daglar-dragomirov/tribes/main/assets/lockup-light.png" alt="TRIBES" width="460">
5
+ </picture>
6
+ </p>
7
+
8
+ <p align="center">
9
+ Inference-time answer selection for optimized DSPy programs
10
+ </p>
11
+
12
+ <p align="center">
13
+ <a href="https://daglar-dragomirov.github.io/tribes/"><b>Website</b></a> &nbsp;|&nbsp;
14
+ <a href="https://daglar-dragomirov.github.io/tribes/guides/quickstart/"><b>Quick Start</b></a> &nbsp;|&nbsp;
15
+ <a href="https://daglar-dragomirov.github.io/tribes/results/"><b>Results</b></a> &nbsp;|&nbsp;
16
+ <a href="research/"><b>Research</b></a> &nbsp;|&nbsp;
17
+ <a href="https://pypi.org/project/tribes/"><b>PyPI</b></a>
18
+ </p>
19
+
20
+ <p align="center">
21
+ <a href="https://pypi.org/project/tribes/"><img src="https://img.shields.io/pypi/v/tribes?logo=pypi&logoColor=white&label=pypi" alt="PyPI"></a>
22
+ <a href="https://pypi.org/project/tribes/"><img src="https://img.shields.io/pypi/pyversions/tribes?logo=python&logoColor=white" alt="Python"></a>
23
+ <a href="https://github.com/daglar-dragomirov/tribes/stargazers"><img src="https://img.shields.io/github/stars/daglar-dragomirov/tribes?logo=github&label=stars" alt="Stars"></a>
24
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License MIT"></a>
25
+ </p>
26
+
27
+ ---
28
+
29
+ ## What is TRIBES?
30
+
31
+ A prompt optimizer such as GEPA, MIPROv2 or SIMBA turns your data into one fixed [DSPy](https://github.com/stanfordnlp/dspy)
32
+ program. That program is good, but it is also brittle: it cannot look anything up, cannot run code to check itself, and
33
+ every answer is a single sample. TRIBES does not touch the optimization. It wraps the finished program with a
34
+ reviewer of its draft and an independent agent, picks one answer per input with a label-free rule, and switches itself
35
+ off on any task where it is not at least as good as the program.
36
+
37
+ ## Install
38
+
39
+ ```bash
40
+ pip install tribes
41
+ ```
42
+
43
+ ## Quickstart
44
+
45
+ ```python
46
+ import dspy
47
+ import tribes
48
+ from tribes import Tribes
49
+
50
+ dspy.configure(lm=dspy.LM("openrouter/z-ai/glm-5.3-flash"))
51
+
52
+ program = dspy.GEPA(metric=metric, ...).compile(student, trainset=train, valset=val) # or MIPROv2, SIMBA, ...
53
+
54
+ pt = Tribes(program, mode="full", examples=train,
55
+ task=tribes.tasks.math(), # optional: how answers compare, checks, tools
56
+ verifier=True) # optional: settle contested items by a check
57
+ pt.fit(train + val, metric) # decides once whether the layer (and the verifier) helps this task
58
+ print(pt.report["use_layer"], pt.report["mean_gain"])
59
+
60
+ prediction = pt(question="...") # same call signature as your program
61
+ ```
62
+
63
+ `fit` only needs labeled data the optimizer already saw; nothing is tuned per dataset.
64
+
65
+ ## How it works
66
+
67
+ ### Three forms
68
+
69
+ | form | what it is |
70
+ |---|---|
71
+ | **rigid** | the optimized program itself, unchanged |
72
+ | **assembled** | the program writes a draft; a ReAct reviewer checks it against the task and fixes it only if it finds a concrete defect |
73
+ | **liquid** | a ReAct agent that solves the task from scratch |
74
+
75
+ Both agents get the same task-agnostic instructions: the program's own instructions (the task's purpose plus what the
76
+ optimizer learned), a few correct outputs from your examples, and one fixed template per form. They have `run_python`
77
+ and any task tools you pass. There is no task-specific prompt text anywhere.
78
+
79
+ ### One answer per input
80
+
81
+ 1. Drop forms whose output is missing or unparsable.
82
+ 2. If you pass a deterministic, label-free `checker` (public unit tests, a SQL query that must run, a plan validator), drop
83
+ forms that fail it. If all fail, keep all.
84
+ 3. If two forms give the same answer, return it. Answers are compared with `answer_key` (default: normalized text).
85
+ 4. Otherwise return the reviewer's answer, then the program's, then the agent's.
86
+
87
+ In `full` mode the program and the reviewer run concurrently, and the agent only when they disagree. `lite` mode runs
88
+ the reviewer only. Every form and every retry is an independent request (its own cache entry), so the vote never
89
+ compares a form with a cached copy of itself.
90
+
91
+ ### Contested inputs: the verifier
92
+
93
+ When no two forms agree, an optional verifier settles the input by evidence, not opinion: it must design and run one
94
+ check whose outcome differs between the candidates (substituting an answer back, code on edge cases, an auxiliary SQL
95
+ query), and code, not the model, accepts a switch only if the quoted evidence matches a tool observation. A switch needs
96
+ two of three votes. `fit` enables the verifier per task only if it wins clearly more items than it loses.
97
+
98
+ ### Task adapters
99
+
100
+ `tribes.tasks` ships what the research used per task type, all label-free: `math()` (answers compare by value),
101
+ `choice()` (option letters), `sql(db_for)` (result-set agreement, a read-only `run_sql` tool, "the query runs"),
102
+ `code(tests_for)` (behavioural agreement on the public-test inputs, `run_public_tests`), `plan(validate)` (stated
103
+ constraints), `tool_qa()` (an abstention is not an answer) and `free_text()`.
104
+
105
+ For outputs that are sets of items scored by recall (retrieved documents, cited sources), `doc_set()` merges instead of
106
+ selecting: an element-wise vote over the forms' sets, capped at the program's own output size. On HoVer this beat
107
+ taking the reviewer's set by about +6 pp on development data for every optimizer.
108
+
109
+ ### Never worse than the program
110
+
111
+ `fit` runs the forms on labeled selection data and compares TRIBES with the program item by item. With the default
112
+ `switch="do_no_harm"` (the 1.0 rule) the layer is used unless it is shown worse by more than one clustered standard
113
+ error; `switch="non_inferiority"` accepts it only if it is shown to be at most `margin` worse. `switch_power` gives the
114
+ operating characteristic of either for your sample size. If the layer is not used, the wrapped object is exactly the
115
+ program, at exactly its cost.
116
+
117
+ ## Results
118
+
119
+ GLM 5.3 Flash as the task model; each optimizer run with its official settings; every number is TRIBES minus that
120
+ optimizer's own program on the same items.
121
+
122
+ **Fresh test (TEST-B, TRIBES 1.1)** — 1,010 new items, used nowhere in development or in the sealed test; the 1.1
123
+ policy and the analysis were pre-registered before any model saw them:
124
+
125
+ | benchmark (test items) | over GEPA | over MIPROv2 | over SIMBA | benchmark mean, 95 % CI |
126
+ |---|---|---|---|---|
127
+ | HoVer (150) | +12.7 | +8.0 | +16.7 | **+12.4** [+8.9, +16.4] |
128
+ | HotpotQA (200) | +7.0 | +3.0 | +2.5 | **+4.2** [+1.8, +6.8] |
129
+ | BIRD (300) | 0.0 | +5.0 | +4.7 | **+3.2** [+0.9, +5.7] |
130
+ | ToolHop (200) | +3.5 | +3.5 | −0.5 | +2.2 [−0.5, +5.0] |
131
+ | GPQA (160) | +0.6 | +4.4 | 0.0 | +1.7 [−1.5, +4.8] |
132
+ | **mean** | +4.8 | +4.8 | +4.7 | **+4.7** [+3.4, +6.1] |
133
+
134
+ Bold: significant (one-sided α = 0.025, Holm over the five benchmarks; the mean: p = 0.0001). Pre-registered
135
+ expectation: about +6 pp. On the same items TRIBES 1.0-lite gains +2.5 pp, so 1.1 adds **+2.2 pp** [+1.5, +3.0] over 1.0.
136
+ On HoVer the set merge beats merging three samples of the program by **+7.1 pp** [+4.4, +10.2]: the gain is not just
137
+ more sampling.
138
+
139
+ **Sealed test (TEST-A)** — opened once, after the rule and every per-task decision were frozen (TRIBES 1.0-lite):
140
+
141
+ | benchmark (test items) | over GEPA | over MIPROv2 | over SIMBA |
142
+ |---|---|---|---|
143
+ | NATURAL PLAN (200) | **+13.5** | **+17.0** | **+10.5** |
144
+ | LiveCodeBench (82) | **+11.0** | **+14.6** | **+13.4** |
145
+ | BIRD (200) | +2.5 | **+9.0** | +3.5 |
146
+ | GPQA (80) | +3.8 | 0.0 | +3.8 |
147
+ | ToolHop (200) | +2.5 | +2.0 | −0.5 (1 item of 200) |
148
+ | **mean** | **+6.7** | **+8.5** | **+6.1** |
149
+
150
+ Bold: 95 % confidence interval above zero. No cell has its interval below zero.
151
+
152
+ **Development** — 13 benchmarks (AIME, GPQA, LiveBench, LiveCodeBench, HotpotQA, HoVer, IFBench, PUPA, BFCL, τ²-bench,
153
+ NATURAL PLAN, BIRD, ToolHop) × 3 optimizers = 36 cells:
154
+
155
+ Per benchmark (TRIBES 1.0 full; bold: 95 % interval above zero; — : not run):
156
+
157
+ | benchmark | over GEPA | over MIPROv2 | over SIMBA |
158
+ |---|---|---|---|
159
+ | AIME | **+29.3** | **+23.3** | **+26.0** |
160
+ | NATURAL PLAN | **+24.0** | **+18.0** | **+21.0** |
161
+ | LiveCodeBench | **+17.0** | **+17.0** | **+18.0** |
162
+ | LiveBench | **+14.8** | **+19.3** | **+13.5** |
163
+ | BIRD | **+18.5** | +2.5 | **+6.5** |
164
+ | HoVer | **+6.7** | +3.7 | **+10.0** |
165
+ | GPQA | +5.1 | **+8.5** | +5.1 |
166
+ | ToolHop | **+8.0** | +0.5 | **+4.5** |
167
+ | HotpotQA | **+4.0** | +3.0 | **+5.7** |
168
+ | IFBench | **+2.2** | 0.0 | **+2.9** |
169
+ | PUPA | +1.7 | 0.0 | **+1.8** |
170
+ | τ²-bench | 0.0 | — | — |
171
+ | BFCL | 0.0 | — | 0.0 |
172
+
173
+ Averages:
174
+
175
+ | | full | lite |
176
+ |---|---|---|
177
+ | over GEPA | +10.1 pp | +9.3 pp |
178
+ | over MIPROv2 | +8.7 pp | +8.4 pp |
179
+ | over SIMBA | +9.6 pp | +8.9 pp |
180
+ | **all 36 cells** | **+9.5 pp** | **+8.9 pp** |
181
+
182
+ No cell below the program; 24 of 36 significantly above. The rule was designed after the development data was seen,
183
+ which is why the two test tables are the ones to quote.
184
+
185
+ **Research record**: how the rule was found, frozen and tested, with every number per cell, is in [`research/`](research/).
186
+
187
+ **Cost**: about 2.6× the program's inference cost in `lite` mode and 4.6× in `full` mode. There is no optimization
188
+ cost.
189
+
190
+ ## API
191
+
192
+ ```python
193
+ Tribes(program, *, mode="full", task=None, tools=(), answer_key=None, checker=None, examples=None,
194
+ verifier=None, switch="do_no_harm", z=None, margin=0.02, parallel=True, independent=True,
195
+ callbacks=(), prices=None, input_fields=None, output_field=None, max_iters=8, retries=3)
196
+ ```
197
+
198
+ | argument | meaning |
199
+ |---|---|
200
+ | `program` | any `dspy.Module`, usually an optimizer's output |
201
+ | `mode` | `"full"`: program + reviewer, agent on disagreement, vote · `"lite"`: reviewer only |
202
+ | `task` | a `tribes.tasks` adapter: answer key, checker, tools and verifier evidence for a task type |
203
+ | `tools` | task tools for both agents and the verifier (`run_python` is always included) |
204
+ | `answer_key` | `callable(prediction) -> hashable`: how two answers are compared |
205
+ | `checker` | `callable(inputs, prediction) -> bool \| None`: a deterministic, label-free check |
206
+ | `verifier` | `True` (or a `Verifier`) to settle contested inputs by an evidence-gated check |
207
+ | `switch`, `z`, `margin` | the task switch: `"do_no_harm"` (1.0) or `"non_inferiority"` |
208
+ | `parallel`, `independent` | run forms concurrently; give each form and retry its own cache entry |
209
+ | `callbacks`, `prices` | events (`LoggingCallback`, `JSONLCallback`, your own); `{model: (usd/M in, usd/M out)}` |
210
+
211
+ | method | what it does |
212
+ |---|---|
213
+ | `fit(selection_set, metric, num_threads=8, cluster=None)` | the task decision; `report` holds the evidence |
214
+ | `pt(**inputs)`, `await pt.acall(**inputs)` | one prediction |
215
+ | `explain(**inputs)` | a `Selection`: every form's output, keys, checks, the verifier's votes |
216
+ | `run_batch(items)`, `await abatch(items)` | many inputs, in order |
217
+ | `cost_report()` | calls, failures, seconds, tokens and dollars per form; cost relative to the program |
218
+ | `save_fit(path)`, `load_fit(path)` | persist the task decision |
219
+
220
+ Command line: `tribes replay rows.jsonl --mode both` applies the rules to saved per-item outputs with no model
221
+ calls; `tribes report fit.json` prints a saved decision.
222
+
223
+ ## FAQ
224
+
225
+ **Does it need a judge model?** No. Earlier versions used an LLM judge; it added +0.8 pp for most of the cost and was
226
+ removed. Agreement between independent forms and cheap deterministic checks do the job.
227
+
228
+ **Which optimizers?** Anything that produces a DSPy program. It was evaluated on GEPA, MIPROv2 and SIMBA.
229
+
230
+ **Is `run_python` a sandbox?** No. It runs code in a subprocess with a 10-second limit. Run untrusted workloads in an
231
+ isolated environment.
232
+
233
+ **What if it does not help my task?** Then `fit` says so (`use_layer = False`) and you get the program back, unchanged.
234
+
235
+ ## Citation
236
+
237
+ ```bibtex
238
+ @software{dragomirov2026tribes,
239
+ author = {Dragomirov, Daglar},
240
+ title = {TRIBES: Inference-time answer selection for optimized DSPy programs},
241
+ year = {2026},
242
+ url = {https://github.com/daglar-dragomirov/tribes}
243
+ }
244
+ ```
245
+
246
+ ## License
247
+
248
+ MIT
@@ -0,0 +1,38 @@
1
+ """TRIBES quickstart: wrap an (optimized) DSPy program, decide on labeled data, then answer.
2
+
3
+ Run: OPENROUTER_API_KEY=... python examples/quickstart.py
4
+ Any LM that DSPy supports works; the model below is only an example.
5
+ """
6
+ import os
7
+
8
+ import dspy
9
+
10
+ from tribes import TRIBES
11
+
12
+ lm = dspy.LM(os.environ.get("PEACHTREES_MODEL", "openrouter/z-ai/glm-5.3-flash"), max_tokens=4096)
13
+ dspy.configure(lm=lm)
14
+
15
+ # 1. Your program. In practice: the program returned by dspy.GEPA / MIPROv2 / SIMBA .compile(...).
16
+ program = dspy.ChainOfThought("question -> answer")
17
+
18
+ # 2. Labeled selection data (the optimizer's train / val sets are fine) and the task metric.
19
+ data = [
20
+ ("A shop sells pens at 3 for $2. How many dollars do 12 pens cost?", "8"),
21
+ ("What is 17 * 23?", "391"),
22
+ ("How many prime numbers are there below 30?", "10"),
23
+ ("A train travels 180 km in 2.5 hours. What is its average speed in km/h?", "72"),
24
+ ("What is the sum of the digits of 2**15?", "26"),
25
+ ("How many days are there in a leap year?", "366"),
26
+ ]
27
+ selection = [dspy.Example(question=q, answer=a).with_inputs("question") for q, a in data]
28
+
29
+
30
+ def metric(example, pred, trace=None):
31
+ return float(str(pred.answer).strip().rstrip(".") == example.answer)
32
+
33
+
34
+ # 3. Wrap, decide once per task, use.
35
+ pt = Tribes(program, mode="lite", examples=selection)
36
+ pt.fit(selection, metric, num_threads=4)
37
+ print({k: v for k, v in pt.report.items() if k != "items"})
38
+ print(pt(question="What is 2**10 + 24?").answer)
@@ -0,0 +1,60 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.25"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "tribes"
7
+ version = "1.1.0"
8
+ description = "Inference-time answer selection for optimized DSPy programs (GEPA, MIPROv2, SIMBA, ...)."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Daglar Dragomirov" }]
14
+ keywords = ["dspy", "llm", "prompt optimization", "gepa", "agents", "react", "selection"]
15
+ classifiers = [
16
+ "Development Status :: 4 - Beta",
17
+ "Intended Audience :: Science/Research",
18
+ "Programming Language :: Python :: 3",
19
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
20
+ "Programming Language :: Python :: 3.10",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Typing :: Typed",
25
+ ]
26
+ dependencies = ["dspy>=3.0", "litellm>=1.60"]
27
+
28
+ [project.urls]
29
+ Homepage = "https://github.com/daglar-dragomirov/tribes"
30
+ Repository = "https://github.com/daglar-dragomirov/tribes"
31
+ Issues = "https://github.com/daglar-dragomirov/tribes/issues"
32
+
33
+ [project.scripts]
34
+ tribes = "tribes.cli:main"
35
+
36
+ [project.optional-dependencies]
37
+ test = ["pytest>=8", "pytest-cov>=5"]
38
+ dev = ["pytest>=8", "pytest-cov>=5", "ruff>=0.6"]
39
+
40
+ [tool.hatch.build.targets.wheel]
41
+ packages = ["src/tribes"]
42
+
43
+ [tool.hatch.build.targets.sdist]
44
+ include = ["src/tribes", "tests", "examples", "README.md", "LICENSE", "pyproject.toml"]
45
+ exclude = ["assets"]
46
+
47
+ [tool.pytest.ini_options]
48
+ testpaths = ["tests"]
49
+ addopts = "-q"
50
+
51
+ [tool.ruff]
52
+ line-length = 125
53
+ target-version = "py310"
54
+
55
+ [tool.ruff.lint]
56
+ select = ["E", "F", "W", "I", "B", "UP"]
57
+ ignore = ["E731", "B008", "B023", "B905", "UP035"]
58
+
59
+ [tool.ruff.lint.per-file-ignores]
60
+ "tests/*" = ["B017"]