tribes 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tribes-1.1.0/.gitignore +11 -0
- tribes-1.1.0/LICENSE +21 -0
- tribes-1.1.0/PKG-INFO +280 -0
- tribes-1.1.0/README.md +248 -0
- tribes-1.1.0/examples/quickstart.py +38 -0
- tribes-1.1.0/pyproject.toml +60 -0
- tribes-1.1.0/research/README.md +49 -0
- tribes-1.1.0/research/fable-replay/README.md +3 -0
- tribes-1.1.0/src/tribes/__init__.py +41 -0
- tribes-1.1.0/src/tribes/callbacks.py +81 -0
- tribes-1.1.0/src/tribes/cli.py +129 -0
- tribes-1.1.0/src/tribes/context.py +63 -0
- tribes-1.1.0/src/tribes/core.py +407 -0
- tribes-1.1.0/src/tribes/forms.py +192 -0
- tribes-1.1.0/src/tribes/py.typed +0 -0
- tribes-1.1.0/src/tribes/replay.py +149 -0
- tribes-1.1.0/src/tribes/select.py +150 -0
- tribes-1.1.0/src/tribes/tasks.py +590 -0
- tribes-1.1.0/src/tribes/usage.py +128 -0
- tribes-1.1.0/src/tribes/verify.py +205 -0
- tribes-1.1.0/tests/conftest.py +61 -0
- tribes-1.1.0/tests/test_core.py +328 -0
- tribes-1.1.0/tests/test_dspy_integration.py +102 -0
- tribes-1.1.0/tests/test_merge.py +67 -0
- tribes-1.1.0/tests/test_replay_cli.py +101 -0
- tribes-1.1.0/tests/test_reproduce.py +45 -0
- tribes-1.1.0/tests/test_select.py +92 -0
- tribes-1.1.0/tests/test_tasks.py +178 -0
- tribes-1.1.0/tests/test_verify.py +151 -0
tribes-1.1.0/.gitignore
ADDED
tribes-1.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Daglar Dragomirov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
tribes-1.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: tribes
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Inference-time answer selection for optimized DSPy programs (GEPA, MIPROv2, SIMBA, ...).
|
|
5
|
+
Project-URL: Homepage, https://github.com/daglar-dragomirov/tribes
|
|
6
|
+
Project-URL: Repository, https://github.com/daglar-dragomirov/tribes
|
|
7
|
+
Project-URL: Issues, https://github.com/daglar-dragomirov/tribes/issues
|
|
8
|
+
Author: Daglar Dragomirov
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agents,dspy,gepa,llm,prompt optimization,react,selection
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: dspy>=3.0
|
|
23
|
+
Requires-Dist: litellm>=1.60
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest-cov>=5; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
27
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
28
|
+
Provides-Extra: test
|
|
29
|
+
Requires-Dist: pytest-cov>=5; extra == 'test'
|
|
30
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
<p align="center">
|
|
34
|
+
<picture>
|
|
35
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/daglar-dragomirov/tribes/main/assets/lockup-dark.png">
|
|
36
|
+
<img src="https://raw.githubusercontent.com/daglar-dragomirov/tribes/main/assets/lockup-light.png" alt="TRIBES" width="460">
|
|
37
|
+
</picture>
|
|
38
|
+
</p>
|
|
39
|
+
|
|
40
|
+
<p align="center">
|
|
41
|
+
Inference-time answer selection for optimized DSPy programs
|
|
42
|
+
</p>
|
|
43
|
+
|
|
44
|
+
<p align="center">
|
|
45
|
+
<a href="https://daglar-dragomirov.github.io/tribes/"><b>Website</b></a> |
|
|
46
|
+
<a href="https://daglar-dragomirov.github.io/tribes/guides/quickstart/"><b>Quick Start</b></a> |
|
|
47
|
+
<a href="https://daglar-dragomirov.github.io/tribes/results/"><b>Results</b></a> |
|
|
48
|
+
<a href="research/"><b>Research</b></a> |
|
|
49
|
+
<a href="https://pypi.org/project/tribes/"><b>PyPI</b></a>
|
|
50
|
+
</p>
|
|
51
|
+
|
|
52
|
+
<p align="center">
|
|
53
|
+
<a href="https://pypi.org/project/tribes/"><img src="https://img.shields.io/pypi/v/tribes?logo=pypi&logoColor=white&label=pypi" alt="PyPI"></a>
|
|
54
|
+
<a href="https://pypi.org/project/tribes/"><img src="https://img.shields.io/pypi/pyversions/tribes?logo=python&logoColor=white" alt="Python"></a>
|
|
55
|
+
<a href="https://github.com/daglar-dragomirov/tribes/stargazers"><img src="https://img.shields.io/github/stars/daglar-dragomirov/tribes?logo=github&label=stars" alt="Stars"></a>
|
|
56
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License MIT"></a>
|
|
57
|
+
</p>
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## What is TRIBES?
|
|
62
|
+
|
|
63
|
+
A prompt optimizer such as GEPA, MIPROv2 or SIMBA turns your data into one fixed [DSPy](https://github.com/stanfordnlp/dspy)
|
|
64
|
+
program. That program is good, but it is also brittle: it cannot look anything up, cannot run code to check itself, and
|
|
65
|
+
every answer is a single sample. TRIBES does not touch the optimization. It wraps the finished program with a
|
|
66
|
+
reviewer of its draft and an independent agent, picks one answer per input with a label-free rule, and switches itself
|
|
67
|
+
off on any task where it is not at least as good as the program.
|
|
68
|
+
|
|
69
|
+
## Install
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pip install tribes
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Quickstart
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import dspy
|
|
79
|
+
import tribes
|
|
80
|
+
from tribes import Tribes
|
|
81
|
+
|
|
82
|
+
dspy.configure(lm=dspy.LM("openrouter/z-ai/glm-5.3-flash"))
|
|
83
|
+
|
|
84
|
+
program = dspy.GEPA(metric=metric, ...).compile(student, trainset=train, valset=val) # or MIPROv2, SIMBA, ...
|
|
85
|
+
|
|
86
|
+
pt = Tribes(program, mode="full", examples=train,
|
|
87
|
+
task=tribes.tasks.math(), # optional: how answers compare, checks, tools
|
|
88
|
+
verifier=True) # optional: settle contested items by a check
|
|
89
|
+
pt.fit(train + val, metric) # decides once whether the layer (and the verifier) helps this task
|
|
90
|
+
print(pt.report["use_layer"], pt.report["mean_gain"])
|
|
91
|
+
|
|
92
|
+
prediction = pt(question="...") # same call signature as your program
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
`fit` only needs labeled data the optimizer already saw; nothing is tuned per dataset.
|
|
96
|
+
|
|
97
|
+
## How it works
|
|
98
|
+
|
|
99
|
+
### Three forms
|
|
100
|
+
|
|
101
|
+
| form | what it is |
|
|
102
|
+
|---|---|
|
|
103
|
+
| **rigid** | the optimized program itself, unchanged |
|
|
104
|
+
| **assembled** | the program writes a draft; a ReAct reviewer checks it against the task and fixes it only if it finds a concrete defect |
|
|
105
|
+
| **liquid** | a ReAct agent that solves the task from scratch |
|
|
106
|
+
|
|
107
|
+
Both agents get the same task-agnostic instructions: the program's own instructions (the task's purpose plus what the
|
|
108
|
+
optimizer learned), a few correct outputs from your examples, and one fixed template per form. They have `run_python`
|
|
109
|
+
and any task tools you pass. There is no task-specific prompt text anywhere.
|
|
110
|
+
|
|
111
|
+
### One answer per input
|
|
112
|
+
|
|
113
|
+
1. Drop forms whose output is missing or unparsable.
|
|
114
|
+
2. If you pass a deterministic, label-free `checker` (public unit tests, a SQL query that must run, a plan validator), drop
|
|
115
|
+
forms that fail it. If all fail, keep all.
|
|
116
|
+
3. If two forms give the same answer, return it. Answers are compared with `answer_key` (default: normalized text).
|
|
117
|
+
4. Otherwise return the reviewer's answer, then the program's, then the agent's.
|
|
118
|
+
|
|
119
|
+
In `full` mode the program and the reviewer run concurrently, and the agent only when they disagree. `lite` mode runs
|
|
120
|
+
the reviewer only. Every form and every retry is an independent request (its own cache entry), so the vote never
|
|
121
|
+
compares a form with a cached copy of itself.
|
|
122
|
+
|
|
123
|
+
### Contested inputs: the verifier
|
|
124
|
+
|
|
125
|
+
When no two forms agree, an optional verifier settles the input by evidence, not opinion: it must design and run one
|
|
126
|
+
check whose outcome differs between the candidates (substituting an answer back, code on edge cases, an auxiliary SQL
|
|
127
|
+
query), and code, not the model, accepts a switch only if the quoted evidence matches a tool observation. A switch needs
|
|
128
|
+
two of three votes. `fit` enables the verifier per task only if it wins clearly more items than it loses.
|
|
129
|
+
|
|
130
|
+
### Task adapters
|
|
131
|
+
|
|
132
|
+
`tribes.tasks` ships what the research used per task type, all label-free: `math()` (answers compare by value),
|
|
133
|
+
`choice()` (option letters), `sql(db_for)` (result-set agreement, a read-only `run_sql` tool, "the query runs"),
|
|
134
|
+
`code(tests_for)` (behavioural agreement on the public-test inputs, `run_public_tests`), `plan(validate)` (stated
|
|
135
|
+
constraints), `tool_qa()` (an abstention is not an answer) and `free_text()`.
|
|
136
|
+
|
|
137
|
+
For outputs that are sets of items scored by recall (retrieved documents, cited sources), `doc_set()` merges instead of
|
|
138
|
+
selecting: an element-wise vote over the forms' sets, capped at the program's own output size. On HoVer this beat
|
|
139
|
+
taking the reviewer's set by about +6 pp on development data for every optimizer.
|
|
140
|
+
|
|
141
|
+
### Never worse than the program
|
|
142
|
+
|
|
143
|
+
`fit` runs the forms on labeled selection data and compares TRIBES with the program item by item. With the default
|
|
144
|
+
`switch="do_no_harm"` (the 1.0 rule) the layer is used unless it is shown worse by more than one clustered standard
|
|
145
|
+
error; `switch="non_inferiority"` accepts it only if it is shown to be at most `margin` worse. `switch_power` gives the
|
|
146
|
+
operating characteristic of either for your sample size. If the layer is not used, the wrapped object is exactly the
|
|
147
|
+
program, at exactly its cost.
|
|
148
|
+
|
|
149
|
+
## Results
|
|
150
|
+
|
|
151
|
+
GLM 5.3 Flash as the task model; each optimizer run with its official settings; every number is TRIBES minus that
|
|
152
|
+
optimizer's own program on the same items.
|
|
153
|
+
|
|
154
|
+
**Fresh test (TEST-B, TRIBES 1.1)** — 1,010 new items, used nowhere in development or in the sealed test; the 1.1
|
|
155
|
+
policy and the analysis were pre-registered before any model saw them:
|
|
156
|
+
|
|
157
|
+
| benchmark (test items) | over GEPA | over MIPROv2 | over SIMBA | benchmark mean, 95 % CI |
|
|
158
|
+
|---|---|---|---|---|
|
|
159
|
+
| HoVer (150) | +12.7 | +8.0 | +16.7 | **+12.4** [+8.9, +16.4] |
|
|
160
|
+
| HotpotQA (200) | +7.0 | +3.0 | +2.5 | **+4.2** [+1.8, +6.8] |
|
|
161
|
+
| BIRD (300) | 0.0 | +5.0 | +4.7 | **+3.2** [+0.9, +5.7] |
|
|
162
|
+
| ToolHop (200) | +3.5 | +3.5 | −0.5 | +2.2 [−0.5, +5.0] |
|
|
163
|
+
| GPQA (160) | +0.6 | +4.4 | 0.0 | +1.7 [−1.5, +4.8] |
|
|
164
|
+
| **mean** | +4.8 | +4.8 | +4.7 | **+4.7** [+3.4, +6.1] |
|
|
165
|
+
|
|
166
|
+
Bold: significant (one-sided α = 0.025, Holm over the five benchmarks; the mean: p = 0.0001). Pre-registered
|
|
167
|
+
expectation: about +6 pp. On the same items TRIBES 1.0-lite gains +2.5 pp, so 1.1 adds **+2.2 pp** [+1.5, +3.0] over 1.0.
|
|
168
|
+
On HoVer the set merge beats merging three samples of the program by **+7.1 pp** [+4.4, +10.2]: the gain is not just
|
|
169
|
+
more sampling.
|
|
170
|
+
|
|
171
|
+
**Sealed test (TEST-A)** — opened once, after the rule and every per-task decision were frozen (TRIBES 1.0-lite):
|
|
172
|
+
|
|
173
|
+
| benchmark (test items) | over GEPA | over MIPROv2 | over SIMBA |
|
|
174
|
+
|---|---|---|---|
|
|
175
|
+
| NATURAL PLAN (200) | **+13.5** | **+17.0** | **+10.5** |
|
|
176
|
+
| LiveCodeBench (82) | **+11.0** | **+14.6** | **+13.4** |
|
|
177
|
+
| BIRD (200) | +2.5 | **+9.0** | +3.5 |
|
|
178
|
+
| GPQA (80) | +3.8 | 0.0 | +3.8 |
|
|
179
|
+
| ToolHop (200) | +2.5 | +2.0 | −0.5 (1 item of 200) |
|
|
180
|
+
| **mean** | **+6.7** | **+8.5** | **+6.1** |
|
|
181
|
+
|
|
182
|
+
Bold: 95 % confidence interval above zero. No cell has its interval below zero.
|
|
183
|
+
|
|
184
|
+
**Development** — 13 benchmarks (AIME, GPQA, LiveBench, LiveCodeBench, HotpotQA, HoVer, IFBench, PUPA, BFCL, τ²-bench,
|
|
185
|
+
NATURAL PLAN, BIRD, ToolHop) × 3 optimizers = 36 cells:
|
|
186
|
+
|
|
187
|
+
Per benchmark (TRIBES 1.0 full; bold: 95 % interval above zero; — : not run):
|
|
188
|
+
|
|
189
|
+
| benchmark | over GEPA | over MIPROv2 | over SIMBA |
|
|
190
|
+
|---|---|---|---|
|
|
191
|
+
| AIME | **+29.3** | **+23.3** | **+26.0** |
|
|
192
|
+
| NATURAL PLAN | **+24.0** | **+18.0** | **+21.0** |
|
|
193
|
+
| LiveCodeBench | **+17.0** | **+17.0** | **+18.0** |
|
|
194
|
+
| LiveBench | **+14.8** | **+19.3** | **+13.5** |
|
|
195
|
+
| BIRD | **+18.5** | +2.5 | **+6.5** |
|
|
196
|
+
| HoVer | **+6.7** | +3.7 | **+10.0** |
|
|
197
|
+
| GPQA | +5.1 | **+8.5** | +5.1 |
|
|
198
|
+
| ToolHop | **+8.0** | +0.5 | **+4.5** |
|
|
199
|
+
| HotpotQA | **+4.0** | +3.0 | **+5.7** |
|
|
200
|
+
| IFBench | **+2.2** | 0.0 | **+2.9** |
|
|
201
|
+
| PUPA | +1.7 | 0.0 | **+1.8** |
|
|
202
|
+
| τ²-bench | 0.0 | — | — |
|
|
203
|
+
| BFCL | 0.0 | — | 0.0 |
|
|
204
|
+
|
|
205
|
+
Averages:
|
|
206
|
+
|
|
207
|
+
| | full | lite |
|
|
208
|
+
|---|---|---|
|
|
209
|
+
| over GEPA | +10.1 pp | +9.3 pp |
|
|
210
|
+
| over MIPROv2 | +8.7 pp | +8.4 pp |
|
|
211
|
+
| over SIMBA | +9.6 pp | +8.9 pp |
|
|
212
|
+
| **all 36 cells** | **+9.5 pp** | **+8.9 pp** |
|
|
213
|
+
|
|
214
|
+
No cell below the program; 24 of 36 significantly above. The rule was designed after the development data was seen,
|
|
215
|
+
which is why the two test tables are the ones to quote.
|
|
216
|
+
|
|
217
|
+
**Research record**: how the rule was found, frozen and tested, with every number per cell, is in [`research/`](research/).
|
|
218
|
+
|
|
219
|
+
**Cost**: about 2.6× the program's inference cost in `lite` mode and 4.6× in `full` mode. There is no optimization
|
|
220
|
+
cost.
|
|
221
|
+
|
|
222
|
+
## API
|
|
223
|
+
|
|
224
|
+
```python
|
|
225
|
+
Tribes(program, *, mode="full", task=None, tools=(), answer_key=None, checker=None, examples=None,
|
|
226
|
+
verifier=None, switch="do_no_harm", z=None, margin=0.02, parallel=True, independent=True,
|
|
227
|
+
callbacks=(), prices=None, input_fields=None, output_field=None, max_iters=8, retries=3)
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
| argument | meaning |
|
|
231
|
+
|---|---|
|
|
232
|
+
| `program` | any `dspy.Module`, usually an optimizer's output |
|
|
233
|
+
| `mode` | `"full"`: program + reviewer, agent on disagreement, vote · `"lite"`: reviewer only |
|
|
234
|
+
| `task` | a `tribes.tasks` adapter: answer key, checker, tools and verifier evidence for a task type |
|
|
235
|
+
| `tools` | task tools for both agents and the verifier (`run_python` is always included) |
|
|
236
|
+
| `answer_key` | `callable(prediction) -> hashable`: how two answers are compared |
|
|
237
|
+
| `checker` | `callable(inputs, prediction) -> bool \| None`: a deterministic, label-free check |
|
|
238
|
+
| `verifier` | `True` (or a `Verifier`) to settle contested inputs by an evidence-gated check |
|
|
239
|
+
| `switch`, `z`, `margin` | the task switch: `"do_no_harm"` (1.0) or `"non_inferiority"` |
|
|
240
|
+
| `parallel`, `independent` | run forms concurrently; give each form and retry its own cache entry |
|
|
241
|
+
| `callbacks`, `prices` | events (`LoggingCallback`, `JSONLCallback`, your own); `{model: (usd/M in, usd/M out)}` |
|
|
242
|
+
|
|
243
|
+
| method | what it does |
|
|
244
|
+
|---|---|
|
|
245
|
+
| `fit(selection_set, metric, num_threads=8, cluster=None)` | the task decision; `report` holds the evidence |
|
|
246
|
+
| `pt(**inputs)`, `await pt.acall(**inputs)` | one prediction |
|
|
247
|
+
| `explain(**inputs)` | a `Selection`: every form's output, keys, checks, the verifier's votes |
|
|
248
|
+
| `run_batch(items)`, `await abatch(items)` | many inputs, in order |
|
|
249
|
+
| `cost_report()` | calls, failures, seconds, tokens and dollars per form; cost relative to the program |
|
|
250
|
+
| `save_fit(path)`, `load_fit(path)` | persist the task decision |
|
|
251
|
+
|
|
252
|
+
Command line: `tribes replay rows.jsonl --mode both` applies the rules to saved per-item outputs with no model
|
|
253
|
+
calls; `tribes report fit.json` prints a saved decision.
|
|
254
|
+
|
|
255
|
+
## FAQ
|
|
256
|
+
|
|
257
|
+
**Does it need a judge model?** No. Earlier versions used an LLM judge; it added +0.8 pp for most of the cost and was
|
|
258
|
+
removed. Agreement between independent forms and cheap deterministic checks do the job.
|
|
259
|
+
|
|
260
|
+
**Which optimizers?** Anything that produces a DSPy program. It was evaluated on GEPA, MIPROv2 and SIMBA.
|
|
261
|
+
|
|
262
|
+
**Is `run_python` a sandbox?** No. It runs code in a subprocess with a 10-second limit. Run untrusted workloads in an
|
|
263
|
+
isolated environment.
|
|
264
|
+
|
|
265
|
+
**What if it does not help my task?** Then `fit` says so (`use_layer = False`) and you get the program back, unchanged.
|
|
266
|
+
|
|
267
|
+
## Citation
|
|
268
|
+
|
|
269
|
+
```bibtex
|
|
270
|
+
@software{dragomirov2026tribes,
|
|
271
|
+
author = {Dragomirov, Daglar},
|
|
272
|
+
title = {TRIBES: Inference-time answer selection for optimized DSPy programs},
|
|
273
|
+
year = {2026},
|
|
274
|
+
url = {https://github.com/daglar-dragomirov/tribes}
|
|
275
|
+
}
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
## License
|
|
279
|
+
|
|
280
|
+
MIT
|
tribes-1.1.0/README.md
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<picture>
|
|
3
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/daglar-dragomirov/tribes/main/assets/lockup-dark.png">
|
|
4
|
+
<img src="https://raw.githubusercontent.com/daglar-dragomirov/tribes/main/assets/lockup-light.png" alt="TRIBES" width="460">
|
|
5
|
+
</picture>
|
|
6
|
+
</p>
|
|
7
|
+
|
|
8
|
+
<p align="center">
|
|
9
|
+
Inference-time answer selection for optimized DSPy programs
|
|
10
|
+
</p>
|
|
11
|
+
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://daglar-dragomirov.github.io/tribes/"><b>Website</b></a> |
|
|
14
|
+
<a href="https://daglar-dragomirov.github.io/tribes/guides/quickstart/"><b>Quick Start</b></a> |
|
|
15
|
+
<a href="https://daglar-dragomirov.github.io/tribes/results/"><b>Results</b></a> |
|
|
16
|
+
<a href="research/"><b>Research</b></a> |
|
|
17
|
+
<a href="https://pypi.org/project/tribes/"><b>PyPI</b></a>
|
|
18
|
+
</p>
|
|
19
|
+
|
|
20
|
+
<p align="center">
|
|
21
|
+
<a href="https://pypi.org/project/tribes/"><img src="https://img.shields.io/pypi/v/tribes?logo=pypi&logoColor=white&label=pypi" alt="PyPI"></a>
|
|
22
|
+
<a href="https://pypi.org/project/tribes/"><img src="https://img.shields.io/pypi/pyversions/tribes?logo=python&logoColor=white" alt="Python"></a>
|
|
23
|
+
<a href="https://github.com/daglar-dragomirov/tribes/stargazers"><img src="https://img.shields.io/github/stars/daglar-dragomirov/tribes?logo=github&label=stars" alt="Stars"></a>
|
|
24
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License MIT"></a>
|
|
25
|
+
</p>
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## What is TRIBES?
|
|
30
|
+
|
|
31
|
+
A prompt optimizer such as GEPA, MIPROv2 or SIMBA turns your data into one fixed [DSPy](https://github.com/stanfordnlp/dspy)
|
|
32
|
+
program. That program is good, but it is also brittle: it cannot look anything up, cannot run code to check itself, and
|
|
33
|
+
every answer is a single sample. TRIBES does not touch the optimization. It wraps the finished program with a
|
|
34
|
+
reviewer of its draft and an independent agent, picks one answer per input with a label-free rule, and switches itself
|
|
35
|
+
off on any task where it is not at least as good as the program.
|
|
36
|
+
|
|
37
|
+
## Install
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install tribes
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Quickstart
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import dspy
|
|
47
|
+
import tribes
|
|
48
|
+
from tribes import Tribes
|
|
49
|
+
|
|
50
|
+
dspy.configure(lm=dspy.LM("openrouter/z-ai/glm-5.3-flash"))
|
|
51
|
+
|
|
52
|
+
program = dspy.GEPA(metric=metric, ...).compile(student, trainset=train, valset=val) # or MIPROv2, SIMBA, ...
|
|
53
|
+
|
|
54
|
+
pt = Tribes(program, mode="full", examples=train,
|
|
55
|
+
task=tribes.tasks.math(), # optional: how answers compare, checks, tools
|
|
56
|
+
verifier=True) # optional: settle contested items by a check
|
|
57
|
+
pt.fit(train + val, metric) # decides once whether the layer (and the verifier) helps this task
|
|
58
|
+
print(pt.report["use_layer"], pt.report["mean_gain"])
|
|
59
|
+
|
|
60
|
+
prediction = pt(question="...") # same call signature as your program
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`fit` only needs labeled data the optimizer already saw; nothing is tuned per dataset.
|
|
64
|
+
|
|
65
|
+
## How it works
|
|
66
|
+
|
|
67
|
+
### Three forms
|
|
68
|
+
|
|
69
|
+
| form | what it is |
|
|
70
|
+
|---|---|
|
|
71
|
+
| **rigid** | the optimized program itself, unchanged |
|
|
72
|
+
| **assembled** | the program writes a draft; a ReAct reviewer checks it against the task and fixes it only if it finds a concrete defect |
|
|
73
|
+
| **liquid** | a ReAct agent that solves the task from scratch |
|
|
74
|
+
|
|
75
|
+
Both agents get the same task-agnostic instructions: the program's own instructions (the task's purpose plus what the
|
|
76
|
+
optimizer learned), a few correct outputs from your examples, and one fixed template per form. They have `run_python`
|
|
77
|
+
and any task tools you pass. There is no task-specific prompt text anywhere.
|
|
78
|
+
|
|
79
|
+
### One answer per input
|
|
80
|
+
|
|
81
|
+
1. Drop forms whose output is missing or unparsable.
|
|
82
|
+
2. If you pass a deterministic, label-free `checker` (public unit tests, a SQL query that must run, a plan validator), drop
|
|
83
|
+
forms that fail it. If all fail, keep all.
|
|
84
|
+
3. If two forms give the same answer, return it. Answers are compared with `answer_key` (default: normalized text).
|
|
85
|
+
4. Otherwise return the reviewer's answer, then the program's, then the agent's.
|
|
86
|
+
|
|
87
|
+
In `full` mode the program and the reviewer run concurrently, and the agent only when they disagree. `lite` mode runs
|
|
88
|
+
the reviewer only. Every form and every retry is an independent request (its own cache entry), so the vote never
|
|
89
|
+
compares a form with a cached copy of itself.
|
|
90
|
+
|
|
91
|
+
### Contested inputs: the verifier
|
|
92
|
+
|
|
93
|
+
When no two forms agree, an optional verifier settles the input by evidence, not opinion: it must design and run one
|
|
94
|
+
check whose outcome differs between the candidates (substituting an answer back, code on edge cases, an auxiliary SQL
|
|
95
|
+
query), and code, not the model, accepts a switch only if the quoted evidence matches a tool observation. A switch needs
|
|
96
|
+
two of three votes. `fit` enables the verifier per task only if it wins clearly more items than it loses.
|
|
97
|
+
|
|
98
|
+
### Task adapters
|
|
99
|
+
|
|
100
|
+
`tribes.tasks` ships what the research used per task type, all label-free: `math()` (answers compare by value),
|
|
101
|
+
`choice()` (option letters), `sql(db_for)` (result-set agreement, a read-only `run_sql` tool, "the query runs"),
|
|
102
|
+
`code(tests_for)` (behavioural agreement on the public-test inputs, `run_public_tests`), `plan(validate)` (stated
|
|
103
|
+
constraints), `tool_qa()` (an abstention is not an answer) and `free_text()`.
|
|
104
|
+
|
|
105
|
+
For outputs that are sets of items scored by recall (retrieved documents, cited sources), `doc_set()` merges instead of
|
|
106
|
+
selecting: an element-wise vote over the forms' sets, capped at the program's own output size. On HoVer this beat
|
|
107
|
+
taking the reviewer's set by about +6 pp on development data for every optimizer.
|
|
108
|
+
|
|
109
|
+
### Never worse than the program
|
|
110
|
+
|
|
111
|
+
`fit` runs the forms on labeled selection data and compares TRIBES with the program item by item. With the default
|
|
112
|
+
`switch="do_no_harm"` (the 1.0 rule) the layer is used unless it is shown worse by more than one clustered standard
|
|
113
|
+
error; `switch="non_inferiority"` accepts it only if it is shown to be at most `margin` worse. `switch_power` gives the
|
|
114
|
+
operating characteristic of either for your sample size. If the layer is not used, the wrapped object is exactly the
|
|
115
|
+
program, at exactly its cost.
|
|
116
|
+
|
|
117
|
+
## Results
|
|
118
|
+
|
|
119
|
+
GLM 5.3 Flash as the task model; each optimizer run with its official settings; every number is TRIBES minus that
|
|
120
|
+
optimizer's own program on the same items.
|
|
121
|
+
|
|
122
|
+
**Fresh test (TEST-B, TRIBES 1.1)** — 1,010 new items, used nowhere in development or in the sealed test; the 1.1
|
|
123
|
+
policy and the analysis were pre-registered before any model saw them:
|
|
124
|
+
|
|
125
|
+
| benchmark (test items) | over GEPA | over MIPROv2 | over SIMBA | benchmark mean, 95 % CI |
|
|
126
|
+
|---|---|---|---|---|
|
|
127
|
+
| HoVer (150) | +12.7 | +8.0 | +16.7 | **+12.4** [+8.9, +16.4] |
|
|
128
|
+
| HotpotQA (200) | +7.0 | +3.0 | +2.5 | **+4.2** [+1.8, +6.8] |
|
|
129
|
+
| BIRD (300) | 0.0 | +5.0 | +4.7 | **+3.2** [+0.9, +5.7] |
|
|
130
|
+
| ToolHop (200) | +3.5 | +3.5 | −0.5 | +2.2 [−0.5, +5.0] |
|
|
131
|
+
| GPQA (160) | +0.6 | +4.4 | 0.0 | +1.7 [−1.5, +4.8] |
|
|
132
|
+
| **mean** | +4.8 | +4.8 | +4.7 | **+4.7** [+3.4, +6.1] |
|
|
133
|
+
|
|
134
|
+
Bold: significant (one-sided α = 0.025, Holm over the five benchmarks; the mean: p = 0.0001). Pre-registered
|
|
135
|
+
expectation: about +6 pp. On the same items TRIBES 1.0-lite gains +2.5 pp, so 1.1 adds **+2.2 pp** [+1.5, +3.0] over 1.0.
|
|
136
|
+
On HoVer the set merge beats merging three samples of the program by **+7.1 pp** [+4.4, +10.2]: the gain is not just
|
|
137
|
+
more sampling.
|
|
138
|
+
|
|
139
|
+
**Sealed test (TEST-A)** — opened once, after the rule and every per-task decision were frozen (TRIBES 1.0-lite):
|
|
140
|
+
|
|
141
|
+
| benchmark (test items) | over GEPA | over MIPROv2 | over SIMBA |
|
|
142
|
+
|---|---|---|---|
|
|
143
|
+
| NATURAL PLAN (200) | **+13.5** | **+17.0** | **+10.5** |
|
|
144
|
+
| LiveCodeBench (82) | **+11.0** | **+14.6** | **+13.4** |
|
|
145
|
+
| BIRD (200) | +2.5 | **+9.0** | +3.5 |
|
|
146
|
+
| GPQA (80) | +3.8 | 0.0 | +3.8 |
|
|
147
|
+
| ToolHop (200) | +2.5 | +2.0 | −0.5 (1 item of 200) |
|
|
148
|
+
| **mean** | **+6.7** | **+8.5** | **+6.1** |
|
|
149
|
+
|
|
150
|
+
Bold: 95 % confidence interval above zero. No cell has its interval below zero.
|
|
151
|
+
|
|
152
|
+
**Development** — 13 benchmarks (AIME, GPQA, LiveBench, LiveCodeBench, HotpotQA, HoVer, IFBench, PUPA, BFCL, τ²-bench,
|
|
153
|
+
NATURAL PLAN, BIRD, ToolHop) × 3 optimizers = 36 cells:
|
|
154
|
+
|
|
155
|
+
Per benchmark (TRIBES 1.0 full; bold: 95 % interval above zero; — : not run):
|
|
156
|
+
|
|
157
|
+
| benchmark | over GEPA | over MIPROv2 | over SIMBA |
|
|
158
|
+
|---|---|---|---|
|
|
159
|
+
| AIME | **+29.3** | **+23.3** | **+26.0** |
|
|
160
|
+
| NATURAL PLAN | **+24.0** | **+18.0** | **+21.0** |
|
|
161
|
+
| LiveCodeBench | **+17.0** | **+17.0** | **+18.0** |
|
|
162
|
+
| LiveBench | **+14.8** | **+19.3** | **+13.5** |
|
|
163
|
+
| BIRD | **+18.5** | +2.5 | **+6.5** |
|
|
164
|
+
| HoVer | **+6.7** | +3.7 | **+10.0** |
|
|
165
|
+
| GPQA | +5.1 | **+8.5** | +5.1 |
|
|
166
|
+
| ToolHop | **+8.0** | +0.5 | **+4.5** |
|
|
167
|
+
| HotpotQA | **+4.0** | +3.0 | **+5.7** |
|
|
168
|
+
| IFBench | **+2.2** | 0.0 | **+2.9** |
|
|
169
|
+
| PUPA | +1.7 | 0.0 | **+1.8** |
|
|
170
|
+
| τ²-bench | 0.0 | — | — |
|
|
171
|
+
| BFCL | 0.0 | — | 0.0 |
|
|
172
|
+
|
|
173
|
+
Averages:
|
|
174
|
+
|
|
175
|
+
| | full | lite |
|
|
176
|
+
|---|---|---|
|
|
177
|
+
| over GEPA | +10.1 pp | +9.3 pp |
|
|
178
|
+
| over MIPROv2 | +8.7 pp | +8.4 pp |
|
|
179
|
+
| over SIMBA | +9.6 pp | +8.9 pp |
|
|
180
|
+
| **all 36 cells** | **+9.5 pp** | **+8.9 pp** |
|
|
181
|
+
|
|
182
|
+
No cell below the program; 24 of 36 significantly above. The rule was designed after the development data was seen,
|
|
183
|
+
which is why the two test tables are the ones to quote.
|
|
184
|
+
|
|
185
|
+
**Research record**: how the rule was found, frozen and tested, with every number per cell, is in [`research/`](research/).
|
|
186
|
+
|
|
187
|
+
**Cost**: about 2.6× the program's inference cost in `lite` mode and 4.6× in `full` mode. There is no optimization
|
|
188
|
+
cost.
|
|
189
|
+
|
|
190
|
+
## API
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
Tribes(program, *, mode="full", task=None, tools=(), answer_key=None, checker=None, examples=None,
|
|
194
|
+
verifier=None, switch="do_no_harm", z=None, margin=0.02, parallel=True, independent=True,
|
|
195
|
+
callbacks=(), prices=None, input_fields=None, output_field=None, max_iters=8, retries=3)
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
| argument | meaning |
|
|
199
|
+
|---|---|
|
|
200
|
+
| `program` | any `dspy.Module`, usually an optimizer's output |
|
|
201
|
+
| `mode` | `"full"`: program + reviewer, agent on disagreement, vote · `"lite"`: reviewer only |
|
|
202
|
+
| `task` | a `tribes.tasks` adapter: answer key, checker, tools and verifier evidence for a task type |
|
|
203
|
+
| `tools` | task tools for both agents and the verifier (`run_python` is always included) |
|
|
204
|
+
| `answer_key` | `callable(prediction) -> hashable`: how two answers are compared |
|
|
205
|
+
| `checker` | `callable(inputs, prediction) -> bool \| None`: a deterministic, label-free check |
|
|
206
|
+
| `verifier` | `True` (or a `Verifier`) to settle contested inputs by an evidence-gated check |
|
|
207
|
+
| `switch`, `z`, `margin` | the task switch: `"do_no_harm"` (1.0) or `"non_inferiority"` |
|
|
208
|
+
| `parallel`, `independent` | run forms concurrently; give each form and retry its own cache entry |
|
|
209
|
+
| `callbacks`, `prices` | events (`LoggingCallback`, `JSONLCallback`, your own); `{model: (usd/M in, usd/M out)}` |
|
|
210
|
+
|
|
211
|
+
| method | what it does |
|
|
212
|
+
|---|---|
|
|
213
|
+
| `fit(selection_set, metric, num_threads=8, cluster=None)` | the task decision; `report` holds the evidence |
|
|
214
|
+
| `pt(**inputs)`, `await pt.acall(**inputs)` | one prediction |
|
|
215
|
+
| `explain(**inputs)` | a `Selection`: every form's output, keys, checks, the verifier's votes |
|
|
216
|
+
| `run_batch(items)`, `await abatch(items)` | many inputs, in order |
|
|
217
|
+
| `cost_report()` | calls, failures, seconds, tokens and dollars per form; cost relative to the program |
|
|
218
|
+
| `save_fit(path)`, `load_fit(path)` | persist the task decision |
|
|
219
|
+
|
|
220
|
+
Command line: `tribes replay rows.jsonl --mode both` applies the rules to saved per-item outputs with no model
|
|
221
|
+
calls; `tribes report fit.json` prints a saved decision.
|
|
222
|
+
|
|
223
|
+
## FAQ
|
|
224
|
+
|
|
225
|
+
**Does it need a judge model?** No. Earlier versions used an LLM judge; it added +0.8 pp for most of the cost and was
|
|
226
|
+
removed. Agreement between independent forms and cheap deterministic checks do the job.
|
|
227
|
+
|
|
228
|
+
**Which optimizers?** Anything that produces a DSPy program. It was evaluated on GEPA, MIPROv2 and SIMBA.
|
|
229
|
+
|
|
230
|
+
**Is `run_python` a sandbox?** No. It runs code in a subprocess with a 10-second limit. Run untrusted workloads in an
|
|
231
|
+
isolated environment.
|
|
232
|
+
|
|
233
|
+
**What if it does not help my task?** Then `fit` says so (`use_layer = False`) and you get the program back, unchanged.
|
|
234
|
+
|
|
235
|
+
## Citation
|
|
236
|
+
|
|
237
|
+
```bibtex
|
|
238
|
+
@software{dragomirov2026tribes,
|
|
239
|
+
author = {Dragomirov, Daglar},
|
|
240
|
+
title = {TRIBES: Inference-time answer selection for optimized DSPy programs},
|
|
241
|
+
year = {2026},
|
|
242
|
+
url = {https://github.com/daglar-dragomirov/tribes}
|
|
243
|
+
}
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
## License
|
|
247
|
+
|
|
248
|
+
MIT
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""TRIBES quickstart: wrap an (optimized) DSPy program, decide on labeled data, then answer.
|
|
2
|
+
|
|
3
|
+
Run: OPENROUTER_API_KEY=... python examples/quickstart.py
|
|
4
|
+
Any LM that DSPy supports works; the model below is only an example.
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
import dspy
|
|
9
|
+
|
|
10
|
+
from tribes import TRIBES
|
|
11
|
+
|
|
12
|
+
lm = dspy.LM(os.environ.get("PEACHTREES_MODEL", "openrouter/z-ai/glm-5.3-flash"), max_tokens=4096)
|
|
13
|
+
dspy.configure(lm=lm)
|
|
14
|
+
|
|
15
|
+
# 1. Your program. In practice: the program returned by dspy.GEPA / MIPROv2 / SIMBA .compile(...).
|
|
16
|
+
program = dspy.ChainOfThought("question -> answer")
|
|
17
|
+
|
|
18
|
+
# 2. Labeled selection data (the optimizer's train / val sets are fine) and the task metric.
|
|
19
|
+
data = [
|
|
20
|
+
("A shop sells pens at 3 for $2. How many dollars do 12 pens cost?", "8"),
|
|
21
|
+
("What is 17 * 23?", "391"),
|
|
22
|
+
("How many prime numbers are there below 30?", "10"),
|
|
23
|
+
("A train travels 180 km in 2.5 hours. What is its average speed in km/h?", "72"),
|
|
24
|
+
("What is the sum of the digits of 2**15?", "26"),
|
|
25
|
+
("How many days are there in a leap year?", "366"),
|
|
26
|
+
]
|
|
27
|
+
selection = [dspy.Example(question=q, answer=a).with_inputs("question") for q, a in data]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def metric(example, pred, trace=None):
|
|
31
|
+
return float(str(pred.answer).strip().rstrip(".") == example.answer)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# 3. Wrap, decide once per task, use.
|
|
35
|
+
pt = Tribes(program, mode="lite", examples=selection)
|
|
36
|
+
pt.fit(selection, metric, num_threads=4)
|
|
37
|
+
print({k: v for k, v in pt.report.items() if k != "items"})
|
|
38
|
+
print(pt(question="What is 2**10 + 24?").answer)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tribes"
|
|
7
|
+
version = "1.1.0"
|
|
8
|
+
description = "Inference-time answer selection for optimized DSPy programs (GEPA, MIPROv2, SIMBA, ...)."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Daglar Dragomirov" }]
|
|
14
|
+
keywords = ["dspy", "llm", "prompt optimization", "gepa", "agents", "react", "selection"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
dependencies = ["dspy>=3.0", "litellm>=1.60"]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/daglar-dragomirov/tribes"
|
|
30
|
+
Repository = "https://github.com/daglar-dragomirov/tribes"
|
|
31
|
+
Issues = "https://github.com/daglar-dragomirov/tribes/issues"
|
|
32
|
+
|
|
33
|
+
[project.scripts]
|
|
34
|
+
tribes = "tribes.cli:main"
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
test = ["pytest>=8", "pytest-cov>=5"]
|
|
38
|
+
dev = ["pytest>=8", "pytest-cov>=5", "ruff>=0.6"]
|
|
39
|
+
|
|
40
|
+
[tool.hatch.build.targets.wheel]
|
|
41
|
+
packages = ["src/tribes"]
|
|
42
|
+
|
|
43
|
+
[tool.hatch.build.targets.sdist]
|
|
44
|
+
include = ["src/tribes", "tests", "examples", "README.md", "LICENSE", "pyproject.toml"]
|
|
45
|
+
exclude = ["assets"]
|
|
46
|
+
|
|
47
|
+
[tool.pytest.ini_options]
|
|
48
|
+
testpaths = ["tests"]
|
|
49
|
+
addopts = "-q"
|
|
50
|
+
|
|
51
|
+
[tool.ruff]
|
|
52
|
+
line-length = 125
|
|
53
|
+
target-version = "py310"
|
|
54
|
+
|
|
55
|
+
[tool.ruff.lint]
|
|
56
|
+
select = ["E", "F", "W", "I", "B", "UP"]
|
|
57
|
+
ignore = ["E731", "B008", "B023", "B905", "UP035"]
|
|
58
|
+
|
|
59
|
+
[tool.ruff.lint.per-file-ignores]
|
|
60
|
+
"tests/*" = ["B017"]
|