tribes 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tribes/__init__.py ADDED
@@ -0,0 +1,41 @@
1
+ """TRIBES: an optimizer-agnostic, never-worse layer on top of an optimized DSPy program.
2
+
3
+ The optimized program (from GEPA, MIPROv2, SIMBA, ...) stays as one of three forms; a ReAct reviewer of its draft and a
4
+ ReAct agent are added, and a label-free rule picks one output per input. An evidence-gated verifier can settle the
5
+ contested inputs. A task-level non-inferiority test on labeled selection data switches the layer off where it does not
6
+ help, so the program is never replaced by something worse on that data.
7
+ """
8
+ __version__ = "1.1.0"
9
+
10
+ from . import tasks # noqa: E402
11
+ from .callbacks import Callback, JSONLCallback, LoggingCallback # noqa: E402
12
+ from .context import current_inputs, item_context, item_tool # noqa: E402
13
+ from .core import Selection, Tribes # noqa: E402
14
+
15
+ PeachTrees = Tribes # the name up to 1.0, kept for compatibility
16
+ from .forms import Assembled, Liquid, run_python, steps # noqa: E402
17
+ from .replay import replay, replay_protocol # noqa: E402
18
+ from .select import ( # noqa: E402
19
+ FORMS,
20
+ contested,
21
+ do_no_harm,
22
+ explain_pick,
23
+ non_inferior,
24
+ non_inferiority,
25
+ paired_stats,
26
+ pick,
27
+ survivors,
28
+ switch_power,
29
+ task_switch,
30
+ text_key,
31
+ )
32
+ from .tasks import Task # noqa: E402
33
+ from .usage import Ledger # noqa: E402
34
+ from .verify import Verdict, Verifier, VerifierGate # noqa: E402
35
+
36
+ __all__ = ["Tribes", "PeachTrees", "Selection", "Task", "tasks", "Verifier", "VerifierGate", "Verdict", "Assembled",
37
+ "Liquid",
38
+ "run_python", "steps", "Callback", "LoggingCallback", "JSONLCallback", "Ledger", "current_inputs",
39
+ "item_context", "item_tool", "replay", "replay_protocol", "FORMS", "pick", "explain_pick", "contested",
40
+ "survivors", "non_inferior", "do_no_harm", "non_inferiority", "task_switch", "switch_power", "paired_stats",
41
+ "text_key"]
tribes/callbacks.py ADDED
@@ -0,0 +1,81 @@
1
+ """Events of a TRIBES run.
2
+
3
+ Subclass :class:`Callback` and override what you need; pass instances as ``Tribes(..., callbacks=[...])``.
4
+ Callbacks are called from worker threads; an exception in a callback is logged and never breaks the run. DSPy's own
5
+ callbacks (``dspy.settings.callbacks``) keep working underneath for the individual LM and module calls.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import logging
11
+ import threading
12
+ from typing import IO, Any, Mapping
13
+
14
+ log = logging.getLogger("tribes")
15
+
16
+
17
+ class Callback:
18
+ def on_form_start(self, form: str, inputs: Mapping[str, Any]) -> None:
19
+ """A form (``rigid`` / ``assembled`` / ``liquid``) or the ``verifier`` starts on one input."""
20
+
21
+ def on_form_end(self, form: str, inputs: Mapping[str, Any], prediction: Any, error: BaseException | None,
22
+ usage: Mapping[str, Any]) -> None:
23
+ """A form finished (after its retries). ``prediction`` is None if it failed; ``usage`` has tokens, cost, seconds."""
24
+
25
+ def on_select(self, inputs: Mapping[str, Any], selection: Any) -> None:
26
+ """The item rule (and the verifier, if any) chose a form; ``selection`` is a :class:`tribes.Selection`."""
27
+
28
+ def on_fit_end(self, report: Mapping[str, Any]) -> None:
29
+ """``fit`` decided whether the layer (and the verifier) is used for this task."""
30
+
31
+
32
+ def emit(callbacks, event: str, *args: Any) -> None:
33
+ for cb in callbacks or ():
34
+ try:
35
+ getattr(cb, event)(*args)
36
+ except Exception as e: # never let observability break inference
37
+ log.warning("callback %s.%s failed: %s", type(cb).__name__, event, e)
38
+
39
+
40
+ class LoggingCallback(Callback):
41
+ """Logs every form call and selection to the ``tribes`` logger."""
42
+
43
+ def __init__(self, level: int = logging.INFO):
44
+ self.level = level
45
+
46
+ def on_form_end(self, form, inputs, prediction, error, usage):
47
+ log.log(self.level, "form %-9s %s in %.2fs, %s+%s tokens", form, "ok" if prediction is not None else "FAILED",
48
+ usage.get("seconds", 0.0), usage.get("prompt_tokens", 0), usage.get("completion_tokens", 0))
49
+
50
+ def on_select(self, inputs, selection):
51
+ log.log(self.level, "selected %s (%s)%s", selection.form, selection.reason,
52
+ f", verifier {selection.verifier['choice']}" if selection.verifier else "")
53
+
54
+ def on_fit_end(self, report):
55
+ log.log(self.level, "fit: use_layer=%s gain=%+.4f se=%.4f n=%d", report.get("use_layer"),
56
+ report.get("mean_gain", 0.0), report.get("se", 0.0), report.get("n", 0))
57
+
58
+
59
+ class JSONLCallback(Callback):
60
+ """Appends one JSON line per event to a file: a run log that ``tribes`` tools and notebooks can read back."""
61
+
62
+ def __init__(self, stream: IO[str]):
63
+ self.stream = stream
64
+ self._lock = threading.Lock()
65
+
66
+ def _write(self, row: dict[str, Any]) -> None:
67
+ with self._lock:
68
+ self.stream.write(json.dumps(row, ensure_ascii=False, default=str) + "\n")
69
+ self.stream.flush()
70
+
71
+ def on_form_end(self, form, inputs, prediction, error, usage):
72
+ self._write({"event": "form", "form": form, "ok": prediction is not None,
73
+ "error": None if error is None else repr(error)[:300], **usage})
74
+
75
+ def on_select(self, inputs, selection):
76
+ self._write({"event": "select", "form": selection.form, "reason": selection.reason,
77
+ "keys": {k: None if v is None else str(v) for k, v in selection.keys.items()},
78
+ "verifier": selection.verifier})
79
+
80
+ def on_fit_end(self, report):
81
+ self._write({"event": "fit", **{k: v for k, v in report.items() if k != "items"}})
tribes/cli.py ADDED
@@ -0,0 +1,129 @@
1
+ """Command line: ``tribes replay | report | pick | info``.
2
+
3
+ ``replay`` apply the rules to saved per-item outputs (no model calls) and print the scores and the decision
4
+ ``report`` print a decision saved with ``Tribes.save_fit``
5
+ ``pick`` the item rule for one input, from answer keys given on the command line
6
+ ``info`` version and environment
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import json
12
+ import sys
13
+ from typing import Sequence
14
+
15
+ from . import __version__
16
+ from .replay import format_report, load_rows, replay, replay_protocol
17
+ from .select import FORMS, explain_pick
18
+
19
+
20
+ def _replay(a: argparse.Namespace) -> int:
21
+ rows = load_rows(a.path)
22
+ modes = ["lite", "full"] if a.mode == "both" else [a.mode]
23
+ if a.protocol:
24
+ sel_roles = tuple(a.select.split(","))
25
+ res = {m: replay_protocol(rows, sel_roles, a.report, m, a.z, a.switch, a.margin, not a.no_checks,
26
+ not a.no_verifier) for m in modes}
27
+ if a.json:
28
+ print(json.dumps(res if len(modes) > 1 else res[modes[0]], indent=2))
29
+ return 0
30
+ for m, cells in res.items():
31
+ print(f"mode {m}: switch on {'+'.join(sel_roles)}, gain on {a.report}")
32
+ print(f" {'cell':10s} {'optimizer':9s} {'dataset':10s} use gain 95% CI")
33
+ for c, x in cells.items():
34
+ print(f" {c:10s} {x.get('opt', ''):9s} {x.get('ds', ''):10s} {'yes' if x['use_layer'] else 'no ':3s} "
35
+ f"{100 * x['report_gain']:+6.2f} [{100 * x['ci95'][0]:+.1f}, {100 * x['ci95'][1]:+.1f}]")
36
+ g = [x["report_gain"] for x in cells.values()]
37
+ print(f" mean over {len(g)} cells: {100 * sum(g) / max(1, len(g)):+.2f} pp")
38
+ return 0
39
+ results = {m: replay(rows, m, a.z, use_checks=not a.no_checks, use_verifier=not a.no_verifier, switch=a.switch,
40
+ margin=a.margin) for m in modes}
41
+ if a.json:
42
+ print(json.dumps(results if len(modes) > 1 else results[modes[0]], indent=2))
43
+ else:
44
+ print("\n\n".join(format_report(r) for r in results.values()))
45
+ return 0
46
+
47
+
48
+ def _report(a: argparse.Namespace) -> int:
49
+ with open(a.path, encoding="utf-8") as f:
50
+ d = json.load(f)
51
+ if a.json:
52
+ print(json.dumps({k: v for k, v in d.items() if k != "items"}, indent=2, default=str))
53
+ return 0
54
+ print(f"mode {d.get('mode')} switch {d.get('switch', 'do_no_harm')} z {d.get('z')}")
55
+ print(f"use layer {d.get('use_layer')} (n {d.get('n')}, gain {d.get('mean_gain', 0):+.4f} "
56
+ f"+- {d.get('se', 0):.4f})")
57
+ if "program_score" in d:
58
+ print(f"program {d['program_score']:.4f}")
59
+ print(f"tribes {d['tribes_score']:.4f}")
60
+ if d.get("oracle_score") is not None:
61
+ print(f"oracle {d['oracle_score']:.4f}")
62
+ v = d.get("verifier")
63
+ if v:
64
+ print(f"verifier {'on' if v.get('enabled') else 'off'} won {v.get('won')} lost {v.get('lost')} "
65
+ f"of {v.get('contested')} contested")
66
+ cost = (d.get("cost") or {}).get("total")
67
+ if cost:
68
+ print(f"cost ${cost.get('cost_usd', 0):.4f}, {cost.get('calls')} calls, "
69
+ f"{cost.get('prompt_tokens')}+{cost.get('completion_tokens')} tokens")
70
+ return 0
71
+
72
+
73
+ def _pick(a: argparse.Namespace) -> int:
74
+ keys = {f: getattr(a, f) for f in FORMS}
75
+ keys = {f: (None if v in (None, "", "-") else v) for f, v in keys.items()}
76
+ form, reason = explain_pick(keys, None, use_vote=a.mode == "full")
77
+ print(f"{form} ({reason})")
78
+ return 0
79
+
80
+
81
+ def _info(a: argparse.Namespace) -> int:
82
+ import platform
83
+ try:
84
+ import dspy
85
+ dv = dspy.__version__
86
+ except Exception: # pragma: no cover
87
+ dv = "not installed"
88
+ print(f"tribes {__version__}\ndspy {dv}\npython {platform.python_version()}")
89
+ return 0
90
+
91
+
92
+ def build_parser() -> argparse.ArgumentParser:
93
+ p = argparse.ArgumentParser(prog="tribes", description="Inference-time answer selection for DSPy programs.")
94
+ p.add_argument("--version", action="version", version=f"tribes {__version__}")
95
+ sub = p.add_subparsers(dest="cmd", required=True)
96
+ r = sub.add_parser("replay", help="apply the rules to saved per-item outputs")
97
+ r.add_argument("path")
98
+ r.add_argument("--mode", choices=["full", "lite", "both"], default="full")
99
+ r.add_argument("--z", type=float, default=None, help="default: 1.0 (do_no_harm) or 1.645 (non_inferiority)")
100
+ r.add_argument("--switch", choices=["do_no_harm", "non_inferiority"], default="do_no_harm")
101
+ r.add_argument("--margin", type=float, default=0.02, help="non-inferiority margin (score units)")
102
+ r.add_argument("--no-checks", action="store_true", help="ignore saved checker results")
103
+ r.add_argument("--no-verifier", action="store_true", help="ignore saved verifier choices")
104
+ r.add_argument("--protocol", action="store_true",
105
+ help="per cell: decide on --select roles, report on --report (rows need 'role' and 'cell')")
106
+ r.add_argument("--select", default="PARETO,FIT")
107
+ r.add_argument("--report", default="DEV")
108
+ r.add_argument("--json", action="store_true")
109
+ r.set_defaults(fn=_replay)
110
+ rep = sub.add_parser("report", help="print a decision saved with save_fit")
111
+ rep.add_argument("path")
112
+ rep.add_argument("--json", action="store_true")
113
+ rep.set_defaults(fn=_report)
114
+ pk = sub.add_parser("pick", help="the item rule for one input")
115
+ for f in FORMS:
116
+ pk.add_argument(f"--{f}", default=None, help=f"answer key of the {f} form ('-' = no output)")
117
+ pk.add_argument("--mode", choices=["full", "lite"], default="full")
118
+ pk.set_defaults(fn=_pick)
119
+ sub.add_parser("info", help="version and environment").set_defaults(fn=_info)
120
+ return p
121
+
122
+
123
+ def main(argv: Sequence[str] | None = None) -> int:
124
+ a = build_parser().parse_args(argv)
125
+ return a.fn(a)
126
+
127
+
128
+ if __name__ == "__main__": # pragma: no cover
129
+ sys.exit(main())
tribes/context.py ADDED
@@ -0,0 +1,63 @@
1
+ """The inputs of the item being processed, visible to per-item tools.
2
+
3
+ ReAct tools are fixed when an agent is built, but many task tools depend on the item: the database of a text-to-SQL
4
+ question, the public tests of a coding problem. TRIBES sets the current inputs for the duration of every form and
5
+ verifier call; a tool reads them with :func:`current_inputs`. The value is a context variable, so it is correct under
6
+ threads (TRIBES copies the context into its workers) and under asyncio.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import contextvars
11
+ from contextlib import contextmanager
12
+ from typing import Any, Callable, Iterator, Mapping
13
+
14
+ _CURRENT: contextvars.ContextVar[Mapping[str, Any] | None] = contextvars.ContextVar("tribes_inputs", default=None)
15
+
16
+
17
+ def current_inputs() -> Mapping[str, Any]:
18
+ """The inputs of the item whose forms are running now. Raises outside a TRIBES call."""
19
+ v = _CURRENT.get()
20
+ if v is None:
21
+ raise RuntimeError("no current item: per-item tools work only inside a TRIBES call (or item_context)")
22
+ return v
23
+
24
+
25
+ @contextmanager
26
+ def item_context(inputs: Mapping[str, Any]) -> Iterator[None]:
27
+ """Make ``inputs`` the current item for the enclosed block (used by TRIBES; handy in tests)."""
28
+ token = _CURRENT.set(dict(inputs))
29
+ try:
30
+ yield
31
+ finally:
32
+ _CURRENT.reset(token)
33
+
34
+
35
+ def item_tool(fn: Callable[..., str], name: str | None = None, doc: str | None = None) -> Callable[..., str]:
36
+ """Turn ``fn(inputs, *args)`` into an agent tool ``tool(*args)`` that reads the current item's inputs.
37
+
38
+ >>> def run_sql(inputs, query: str) -> str: ...
39
+ >>> tool = item_tool(run_sql) # the agent sees run_sql(query)
40
+ """
41
+ import functools
42
+ import inspect
43
+
44
+ sig = inspect.signature(fn)
45
+ params = list(sig.parameters.values())[1:]
46
+
47
+ @functools.wraps(fn)
48
+ def tool(*args, **kwargs):
49
+ try:
50
+ return fn(current_inputs(), *args, **kwargs)
51
+ except Exception as e: # a tool never raises into the agent
52
+ return f"error: {type(e).__name__}: {str(e)[:300]}"
53
+
54
+ tool.__signature__ = sig.replace(parameters=params) # type: ignore[attr-defined]
55
+ ann = dict(getattr(fn, "__annotations__", {}))
56
+ first = next(iter(sig.parameters), None)
57
+ ann.pop(first, None)
58
+ tool.__annotations__ = ann
59
+ if name:
60
+ tool.__name__ = tool.__qualname__ = name
61
+ if doc:
62
+ tool.__doc__ = doc
63
+ return tool