tribes 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tribes/__init__.py +41 -0
- tribes/callbacks.py +81 -0
- tribes/cli.py +129 -0
- tribes/context.py +63 -0
- tribes/core.py +407 -0
- tribes/forms.py +192 -0
- tribes/py.typed +0 -0
- tribes/replay.py +149 -0
- tribes/select.py +150 -0
- tribes/tasks.py +590 -0
- tribes/usage.py +128 -0
- tribes/verify.py +205 -0
- tribes-1.1.0.dist-info/METADATA +280 -0
- tribes-1.1.0.dist-info/RECORD +17 -0
- tribes-1.1.0.dist-info/WHEEL +4 -0
- tribes-1.1.0.dist-info/entry_points.txt +2 -0
- tribes-1.1.0.dist-info/licenses/LICENSE +21 -0
tribes/__init__.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""TRIBES: an optimizer-agnostic, never-worse layer on top of an optimized DSPy program.
|
|
2
|
+
|
|
3
|
+
The optimized program (from GEPA, MIPROv2, SIMBA, ...) stays as one of three forms; a ReAct reviewer of its draft and a
|
|
4
|
+
ReAct agent are added, and a label-free rule picks one output per input. An evidence-gated verifier can settle the
|
|
5
|
+
contested inputs. A task-level non-inferiority test on labeled selection data switches the layer off where it does not
|
|
6
|
+
help, so the program is never replaced by something worse on that data.
|
|
7
|
+
"""
|
|
8
|
+
__version__ = "1.1.0"
|
|
9
|
+
|
|
10
|
+
from . import tasks # noqa: E402
|
|
11
|
+
from .callbacks import Callback, JSONLCallback, LoggingCallback # noqa: E402
|
|
12
|
+
from .context import current_inputs, item_context, item_tool # noqa: E402
|
|
13
|
+
from .core import Selection, Tribes # noqa: E402
|
|
14
|
+
|
|
15
|
+
PeachTrees = Tribes # the name up to 1.0, kept for compatibility
|
|
16
|
+
from .forms import Assembled, Liquid, run_python, steps # noqa: E402
|
|
17
|
+
from .replay import replay, replay_protocol # noqa: E402
|
|
18
|
+
from .select import ( # noqa: E402
|
|
19
|
+
FORMS,
|
|
20
|
+
contested,
|
|
21
|
+
do_no_harm,
|
|
22
|
+
explain_pick,
|
|
23
|
+
non_inferior,
|
|
24
|
+
non_inferiority,
|
|
25
|
+
paired_stats,
|
|
26
|
+
pick,
|
|
27
|
+
survivors,
|
|
28
|
+
switch_power,
|
|
29
|
+
task_switch,
|
|
30
|
+
text_key,
|
|
31
|
+
)
|
|
32
|
+
from .tasks import Task # noqa: E402
|
|
33
|
+
from .usage import Ledger # noqa: E402
|
|
34
|
+
from .verify import Verdict, Verifier, VerifierGate # noqa: E402
|
|
35
|
+
|
|
36
|
+
__all__ = ["Tribes", "PeachTrees", "Selection", "Task", "tasks", "Verifier", "VerifierGate", "Verdict", "Assembled",
|
|
37
|
+
"Liquid",
|
|
38
|
+
"run_python", "steps", "Callback", "LoggingCallback", "JSONLCallback", "Ledger", "current_inputs",
|
|
39
|
+
"item_context", "item_tool", "replay", "replay_protocol", "FORMS", "pick", "explain_pick", "contested",
|
|
40
|
+
"survivors", "non_inferior", "do_no_harm", "non_inferiority", "task_switch", "switch_power", "paired_stats",
|
|
41
|
+
"text_key"]
|
tribes/callbacks.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Events of a TRIBES run.
|
|
2
|
+
|
|
3
|
+
Subclass :class:`Callback` and override what you need; pass instances as ``Tribes(..., callbacks=[...])``.
|
|
4
|
+
Callbacks are called from worker threads; an exception in a callback is logged and never breaks the run. DSPy's own
|
|
5
|
+
callbacks (``dspy.settings.callbacks``) keep working underneath for the individual LM and module calls.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
import threading
|
|
12
|
+
from typing import IO, Any, Mapping
|
|
13
|
+
|
|
14
|
+
log = logging.getLogger("tribes")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Callback:
|
|
18
|
+
def on_form_start(self, form: str, inputs: Mapping[str, Any]) -> None:
|
|
19
|
+
"""A form (``rigid`` / ``assembled`` / ``liquid``) or the ``verifier`` starts on one input."""
|
|
20
|
+
|
|
21
|
+
def on_form_end(self, form: str, inputs: Mapping[str, Any], prediction: Any, error: BaseException | None,
|
|
22
|
+
usage: Mapping[str, Any]) -> None:
|
|
23
|
+
"""A form finished (after its retries). ``prediction`` is None if it failed; ``usage`` has tokens, cost, seconds."""
|
|
24
|
+
|
|
25
|
+
def on_select(self, inputs: Mapping[str, Any], selection: Any) -> None:
|
|
26
|
+
"""The item rule (and the verifier, if any) chose a form; ``selection`` is a :class:`tribes.Selection`."""
|
|
27
|
+
|
|
28
|
+
def on_fit_end(self, report: Mapping[str, Any]) -> None:
|
|
29
|
+
"""``fit`` decided whether the layer (and the verifier) is used for this task."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def emit(callbacks, event: str, *args: Any) -> None:
|
|
33
|
+
for cb in callbacks or ():
|
|
34
|
+
try:
|
|
35
|
+
getattr(cb, event)(*args)
|
|
36
|
+
except Exception as e: # never let observability break inference
|
|
37
|
+
log.warning("callback %s.%s failed: %s", type(cb).__name__, event, e)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class LoggingCallback(Callback):
|
|
41
|
+
"""Logs every form call and selection to the ``tribes`` logger."""
|
|
42
|
+
|
|
43
|
+
def __init__(self, level: int = logging.INFO):
|
|
44
|
+
self.level = level
|
|
45
|
+
|
|
46
|
+
def on_form_end(self, form, inputs, prediction, error, usage):
|
|
47
|
+
log.log(self.level, "form %-9s %s in %.2fs, %s+%s tokens", form, "ok" if prediction is not None else "FAILED",
|
|
48
|
+
usage.get("seconds", 0.0), usage.get("prompt_tokens", 0), usage.get("completion_tokens", 0))
|
|
49
|
+
|
|
50
|
+
def on_select(self, inputs, selection):
|
|
51
|
+
log.log(self.level, "selected %s (%s)%s", selection.form, selection.reason,
|
|
52
|
+
f", verifier {selection.verifier['choice']}" if selection.verifier else "")
|
|
53
|
+
|
|
54
|
+
def on_fit_end(self, report):
|
|
55
|
+
log.log(self.level, "fit: use_layer=%s gain=%+.4f se=%.4f n=%d", report.get("use_layer"),
|
|
56
|
+
report.get("mean_gain", 0.0), report.get("se", 0.0), report.get("n", 0))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class JSONLCallback(Callback):
|
|
60
|
+
"""Appends one JSON line per event to a file: a run log that ``tribes`` tools and notebooks can read back."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, stream: IO[str]):
|
|
63
|
+
self.stream = stream
|
|
64
|
+
self._lock = threading.Lock()
|
|
65
|
+
|
|
66
|
+
def _write(self, row: dict[str, Any]) -> None:
|
|
67
|
+
with self._lock:
|
|
68
|
+
self.stream.write(json.dumps(row, ensure_ascii=False, default=str) + "\n")
|
|
69
|
+
self.stream.flush()
|
|
70
|
+
|
|
71
|
+
def on_form_end(self, form, inputs, prediction, error, usage):
|
|
72
|
+
self._write({"event": "form", "form": form, "ok": prediction is not None,
|
|
73
|
+
"error": None if error is None else repr(error)[:300], **usage})
|
|
74
|
+
|
|
75
|
+
def on_select(self, inputs, selection):
|
|
76
|
+
self._write({"event": "select", "form": selection.form, "reason": selection.reason,
|
|
77
|
+
"keys": {k: None if v is None else str(v) for k, v in selection.keys.items()},
|
|
78
|
+
"verifier": selection.verifier})
|
|
79
|
+
|
|
80
|
+
def on_fit_end(self, report):
|
|
81
|
+
self._write({"event": "fit", **{k: v for k, v in report.items() if k != "items"}})
|
tribes/cli.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Command line: ``tribes replay | report | pick | info``.
|
|
2
|
+
|
|
3
|
+
``replay`` apply the rules to saved per-item outputs (no model calls) and print the scores and the decision
|
|
4
|
+
``report`` print a decision saved with ``Tribes.save_fit``
|
|
5
|
+
``pick`` the item rule for one input, from answer keys given on the command line
|
|
6
|
+
``info`` version and environment
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import sys
|
|
13
|
+
from typing import Sequence
|
|
14
|
+
|
|
15
|
+
from . import __version__
|
|
16
|
+
from .replay import format_report, load_rows, replay, replay_protocol
|
|
17
|
+
from .select import FORMS, explain_pick
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _replay(a: argparse.Namespace) -> int:
|
|
21
|
+
rows = load_rows(a.path)
|
|
22
|
+
modes = ["lite", "full"] if a.mode == "both" else [a.mode]
|
|
23
|
+
if a.protocol:
|
|
24
|
+
sel_roles = tuple(a.select.split(","))
|
|
25
|
+
res = {m: replay_protocol(rows, sel_roles, a.report, m, a.z, a.switch, a.margin, not a.no_checks,
|
|
26
|
+
not a.no_verifier) for m in modes}
|
|
27
|
+
if a.json:
|
|
28
|
+
print(json.dumps(res if len(modes) > 1 else res[modes[0]], indent=2))
|
|
29
|
+
return 0
|
|
30
|
+
for m, cells in res.items():
|
|
31
|
+
print(f"mode {m}: switch on {'+'.join(sel_roles)}, gain on {a.report}")
|
|
32
|
+
print(f" {'cell':10s} {'optimizer':9s} {'dataset':10s} use gain 95% CI")
|
|
33
|
+
for c, x in cells.items():
|
|
34
|
+
print(f" {c:10s} {x.get('opt', ''):9s} {x.get('ds', ''):10s} {'yes' if x['use_layer'] else 'no ':3s} "
|
|
35
|
+
f"{100 * x['report_gain']:+6.2f} [{100 * x['ci95'][0]:+.1f}, {100 * x['ci95'][1]:+.1f}]")
|
|
36
|
+
g = [x["report_gain"] for x in cells.values()]
|
|
37
|
+
print(f" mean over {len(g)} cells: {100 * sum(g) / max(1, len(g)):+.2f} pp")
|
|
38
|
+
return 0
|
|
39
|
+
results = {m: replay(rows, m, a.z, use_checks=not a.no_checks, use_verifier=not a.no_verifier, switch=a.switch,
|
|
40
|
+
margin=a.margin) for m in modes}
|
|
41
|
+
if a.json:
|
|
42
|
+
print(json.dumps(results if len(modes) > 1 else results[modes[0]], indent=2))
|
|
43
|
+
else:
|
|
44
|
+
print("\n\n".join(format_report(r) for r in results.values()))
|
|
45
|
+
return 0
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _report(a: argparse.Namespace) -> int:
|
|
49
|
+
with open(a.path, encoding="utf-8") as f:
|
|
50
|
+
d = json.load(f)
|
|
51
|
+
if a.json:
|
|
52
|
+
print(json.dumps({k: v for k, v in d.items() if k != "items"}, indent=2, default=str))
|
|
53
|
+
return 0
|
|
54
|
+
print(f"mode {d.get('mode')} switch {d.get('switch', 'do_no_harm')} z {d.get('z')}")
|
|
55
|
+
print(f"use layer {d.get('use_layer')} (n {d.get('n')}, gain {d.get('mean_gain', 0):+.4f} "
|
|
56
|
+
f"+- {d.get('se', 0):.4f})")
|
|
57
|
+
if "program_score" in d:
|
|
58
|
+
print(f"program {d['program_score']:.4f}")
|
|
59
|
+
print(f"tribes {d['tribes_score']:.4f}")
|
|
60
|
+
if d.get("oracle_score") is not None:
|
|
61
|
+
print(f"oracle {d['oracle_score']:.4f}")
|
|
62
|
+
v = d.get("verifier")
|
|
63
|
+
if v:
|
|
64
|
+
print(f"verifier {'on' if v.get('enabled') else 'off'} won {v.get('won')} lost {v.get('lost')} "
|
|
65
|
+
f"of {v.get('contested')} contested")
|
|
66
|
+
cost = (d.get("cost") or {}).get("total")
|
|
67
|
+
if cost:
|
|
68
|
+
print(f"cost ${cost.get('cost_usd', 0):.4f}, {cost.get('calls')} calls, "
|
|
69
|
+
f"{cost.get('prompt_tokens')}+{cost.get('completion_tokens')} tokens")
|
|
70
|
+
return 0
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _pick(a: argparse.Namespace) -> int:
|
|
74
|
+
keys = {f: getattr(a, f) for f in FORMS}
|
|
75
|
+
keys = {f: (None if v in (None, "", "-") else v) for f, v in keys.items()}
|
|
76
|
+
form, reason = explain_pick(keys, None, use_vote=a.mode == "full")
|
|
77
|
+
print(f"{form} ({reason})")
|
|
78
|
+
return 0
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _info(a: argparse.Namespace) -> int:
|
|
82
|
+
import platform
|
|
83
|
+
try:
|
|
84
|
+
import dspy
|
|
85
|
+
dv = dspy.__version__
|
|
86
|
+
except Exception: # pragma: no cover
|
|
87
|
+
dv = "not installed"
|
|
88
|
+
print(f"tribes {__version__}\ndspy {dv}\npython {platform.python_version()}")
|
|
89
|
+
return 0
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
93
|
+
p = argparse.ArgumentParser(prog="tribes", description="Inference-time answer selection for DSPy programs.")
|
|
94
|
+
p.add_argument("--version", action="version", version=f"tribes {__version__}")
|
|
95
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
96
|
+
r = sub.add_parser("replay", help="apply the rules to saved per-item outputs")
|
|
97
|
+
r.add_argument("path")
|
|
98
|
+
r.add_argument("--mode", choices=["full", "lite", "both"], default="full")
|
|
99
|
+
r.add_argument("--z", type=float, default=None, help="default: 1.0 (do_no_harm) or 1.645 (non_inferiority)")
|
|
100
|
+
r.add_argument("--switch", choices=["do_no_harm", "non_inferiority"], default="do_no_harm")
|
|
101
|
+
r.add_argument("--margin", type=float, default=0.02, help="non-inferiority margin (score units)")
|
|
102
|
+
r.add_argument("--no-checks", action="store_true", help="ignore saved checker results")
|
|
103
|
+
r.add_argument("--no-verifier", action="store_true", help="ignore saved verifier choices")
|
|
104
|
+
r.add_argument("--protocol", action="store_true",
|
|
105
|
+
help="per cell: decide on --select roles, report on --report (rows need 'role' and 'cell')")
|
|
106
|
+
r.add_argument("--select", default="PARETO,FIT")
|
|
107
|
+
r.add_argument("--report", default="DEV")
|
|
108
|
+
r.add_argument("--json", action="store_true")
|
|
109
|
+
r.set_defaults(fn=_replay)
|
|
110
|
+
rep = sub.add_parser("report", help="print a decision saved with save_fit")
|
|
111
|
+
rep.add_argument("path")
|
|
112
|
+
rep.add_argument("--json", action="store_true")
|
|
113
|
+
rep.set_defaults(fn=_report)
|
|
114
|
+
pk = sub.add_parser("pick", help="the item rule for one input")
|
|
115
|
+
for f in FORMS:
|
|
116
|
+
pk.add_argument(f"--{f}", default=None, help=f"answer key of the {f} form ('-' = no output)")
|
|
117
|
+
pk.add_argument("--mode", choices=["full", "lite"], default="full")
|
|
118
|
+
pk.set_defaults(fn=_pick)
|
|
119
|
+
sub.add_parser("info", help="version and environment").set_defaults(fn=_info)
|
|
120
|
+
return p
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
124
|
+
a = build_parser().parse_args(argv)
|
|
125
|
+
return a.fn(a)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
if __name__ == "__main__": # pragma: no cover
|
|
129
|
+
sys.exit(main())
|
tribes/context.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""The inputs of the item being processed, visible to per-item tools.
|
|
2
|
+
|
|
3
|
+
ReAct tools are fixed when an agent is built, but many task tools depend on the item: the database of a text-to-SQL
|
|
4
|
+
question, the public tests of a coding problem. TRIBES sets the current inputs for the duration of every form and
|
|
5
|
+
verifier call; a tool reads them with :func:`current_inputs`. The value is a context variable, so it is correct under
|
|
6
|
+
threads (TRIBES copies the context into its workers) and under asyncio.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import contextvars
|
|
11
|
+
from contextlib import contextmanager
|
|
12
|
+
from typing import Any, Callable, Iterator, Mapping
|
|
13
|
+
|
|
14
|
+
_CURRENT: contextvars.ContextVar[Mapping[str, Any] | None] = contextvars.ContextVar("tribes_inputs", default=None)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def current_inputs() -> Mapping[str, Any]:
|
|
18
|
+
"""The inputs of the item whose forms are running now. Raises outside a TRIBES call."""
|
|
19
|
+
v = _CURRENT.get()
|
|
20
|
+
if v is None:
|
|
21
|
+
raise RuntimeError("no current item: per-item tools work only inside a TRIBES call (or item_context)")
|
|
22
|
+
return v
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@contextmanager
|
|
26
|
+
def item_context(inputs: Mapping[str, Any]) -> Iterator[None]:
|
|
27
|
+
"""Make ``inputs`` the current item for the enclosed block (used by TRIBES; handy in tests)."""
|
|
28
|
+
token = _CURRENT.set(dict(inputs))
|
|
29
|
+
try:
|
|
30
|
+
yield
|
|
31
|
+
finally:
|
|
32
|
+
_CURRENT.reset(token)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def item_tool(fn: Callable[..., str], name: str | None = None, doc: str | None = None) -> Callable[..., str]:
|
|
36
|
+
"""Turn ``fn(inputs, *args)`` into an agent tool ``tool(*args)`` that reads the current item's inputs.
|
|
37
|
+
|
|
38
|
+
>>> def run_sql(inputs, query: str) -> str: ...
|
|
39
|
+
>>> tool = item_tool(run_sql) # the agent sees run_sql(query)
|
|
40
|
+
"""
|
|
41
|
+
import functools
|
|
42
|
+
import inspect
|
|
43
|
+
|
|
44
|
+
sig = inspect.signature(fn)
|
|
45
|
+
params = list(sig.parameters.values())[1:]
|
|
46
|
+
|
|
47
|
+
@functools.wraps(fn)
|
|
48
|
+
def tool(*args, **kwargs):
|
|
49
|
+
try:
|
|
50
|
+
return fn(current_inputs(), *args, **kwargs)
|
|
51
|
+
except Exception as e: # a tool never raises into the agent
|
|
52
|
+
return f"error: {type(e).__name__}: {str(e)[:300]}"
|
|
53
|
+
|
|
54
|
+
tool.__signature__ = sig.replace(parameters=params) # type: ignore[attr-defined]
|
|
55
|
+
ann = dict(getattr(fn, "__annotations__", {}))
|
|
56
|
+
first = next(iter(sig.parameters), None)
|
|
57
|
+
ann.pop(first, None)
|
|
58
|
+
tool.__annotations__ = ann
|
|
59
|
+
if name:
|
|
60
|
+
tool.__name__ = tool.__qualname__ = name
|
|
61
|
+
if doc:
|
|
62
|
+
tool.__doc__ = doc
|
|
63
|
+
return tool
|