rulesmith 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rulesmith/__init__.py +1 -0
- rulesmith/ablate.py +99 -0
- rulesmith/arena.py +172 -0
- rulesmith/bench.py +1068 -0
- rulesmith/calibrate.py +457 -0
- rulesmith/chat_judge.py +205 -0
- rulesmith/chess.py +526 -0
- rulesmith/clef.py +66 -0
- rulesmith/cli.py +1234 -0
- rulesmith/diagram.py +226 -0
- rulesmith/doom.py +550 -0
- rulesmith/extract.py +77 -0
- rulesmith/grade.py +85 -0
- rulesmith/graph.py +975 -0
- rulesmith/label.py +67 -0
- rulesmith/level.py +389 -0
- rulesmith/maps.py +96 -0
- rulesmith/mine.py +313 -0
- rulesmith/optimize.py +931 -0
- rulesmith/rules.py +1017 -0
- rulesmith/runtime.py +711 -0
- rulesmith/serve.py +68 -0
- rulesmith/tuning.py +134 -0
- rulesmith-0.1.0.dist-info/METADATA +131 -0
- rulesmith-0.1.0.dist-info/RECORD +28 -0
- rulesmith-0.1.0.dist-info/WHEEL +4 -0
- rulesmith-0.1.0.dist-info/entry_points.txt +2 -0
- rulesmith-0.1.0.dist-info/licenses/LICENSE +21 -0
rulesmith/diagram.py
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""A plan as a Mermaid flowchart, for looking at a graph rules cannot express."""
|
|
2
|
+
|
|
3
|
+
from textwrap import shorten
|
|
4
|
+
|
|
5
|
+
from rulesmith.graph import (
|
|
6
|
+
Call,
|
|
7
|
+
Coalesce,
|
|
8
|
+
Confidence,
|
|
9
|
+
Extract,
|
|
10
|
+
Graph,
|
|
11
|
+
Merge,
|
|
12
|
+
NodeInput,
|
|
13
|
+
Plan,
|
|
14
|
+
Present,
|
|
15
|
+
Quantify,
|
|
16
|
+
Read,
|
|
17
|
+
Sum,
|
|
18
|
+
Switch,
|
|
19
|
+
Threshold,
|
|
20
|
+
Value,
|
|
21
|
+
Vote,
|
|
22
|
+
tested,
|
|
23
|
+
)
|
|
24
|
+
from rulesmith.rules import (
|
|
25
|
+
GENERATED,
|
|
26
|
+
name_of,
|
|
27
|
+
number,
|
|
28
|
+
path_of,
|
|
29
|
+
render_condition,
|
|
30
|
+
render_quantifier,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
SHAPES = {
|
|
34
|
+
Read: ("[/", "/]"),
|
|
35
|
+
Call: ("{{", "}}"),
|
|
36
|
+
Extract: ("{{", "}}"),
|
|
37
|
+
Threshold: ("[", "]"),
|
|
38
|
+
Switch: ("[", "]"),
|
|
39
|
+
Merge: ("[", "]"),
|
|
40
|
+
Coalesce: ("([", "])"),
|
|
41
|
+
Vote: ("[/", "\\]"),
|
|
42
|
+
Confidence: ("[", "]"),
|
|
43
|
+
Sum: ("[", "]"),
|
|
44
|
+
Quantify: ("[", "]"),
|
|
45
|
+
Value: ("((", "))"),
|
|
46
|
+
}
|
|
47
|
+
# Colors by what a node costs to run, with dark text so each reads on a light or dark page: the
|
|
48
|
+
# judge is the one place a model is asked, whichever backend runs the graph.
|
|
49
|
+
KINDS = {
|
|
50
|
+
"judge": (Call, Extract),
|
|
51
|
+
"read": (Read,),
|
|
52
|
+
"rule": (Value, Threshold),
|
|
53
|
+
"unit": (Graph,),
|
|
54
|
+
}
|
|
55
|
+
STYLES = {
|
|
56
|
+
"judge": "fill:#fcd34d,stroke:#b45309,stroke-width:2px,color:#1c1917",
|
|
57
|
+
"unit": "fill:#ede9fe,stroke:#7c3aed,color:#1c1917",
|
|
58
|
+
"condition": "fill:#e5e7eb,stroke:#6b7280,color:#1c1917",
|
|
59
|
+
"read": "fill:#bfdbfe,stroke:#1d4ed8,color:#1c1917",
|
|
60
|
+
"rule": "fill:#bbf7d0,stroke:#15803d,color:#1c1917",
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def label(name: str, node, reads: dict[str, Read], plan: Plan) -> str:
|
|
65
|
+
"""What a box says: enough to recognize the node, not its whole definition."""
|
|
66
|
+
if isinstance(node, Read):
|
|
67
|
+
return f"read {path_of(node)}"
|
|
68
|
+
if isinstance(node, Call):
|
|
69
|
+
# Named as the rules test it, and marked as the model call it is.
|
|
70
|
+
named = "judge" if GENERATED.fullmatch(name) else f"judge: {name}"
|
|
71
|
+
named += f", asked of {node.judge}" if node.judge else ""
|
|
72
|
+
question = shorten(node.question.instructions, 48, placeholder="...")
|
|
73
|
+
return f"{named}<br/>{node.question.type}: {question}"
|
|
74
|
+
if isinstance(node, Extract):
|
|
75
|
+
# A model call too, so it wears the judge's color; what it does is read, not judge.
|
|
76
|
+
return f"extract: {name}<br/>{shorten(node.instructions, 48, placeholder='...')}"
|
|
77
|
+
if isinstance(node, Threshold):
|
|
78
|
+
return f"{name}: >= {number(node.cutoff)} ? {node.above} : {node.below}"
|
|
79
|
+
if isinstance(node, Value):
|
|
80
|
+
# A named condition holds a bare true; its name is what the rules test.
|
|
81
|
+
return name if node.value is True else str(node.value)
|
|
82
|
+
if isinstance(node, Switch):
|
|
83
|
+
return f"{name}: switch on {node.source}"
|
|
84
|
+
if isinstance(node, Vote):
|
|
85
|
+
return f"{name}: {node.rule} of {len(node.sources)}"
|
|
86
|
+
if isinstance(node, Coalesce):
|
|
87
|
+
return f"{name}<br/>first that answers"
|
|
88
|
+
if isinstance(node, Confidence):
|
|
89
|
+
return f"how sure {node.source} was"
|
|
90
|
+
if isinstance(node, Sum):
|
|
91
|
+
return f"{name}: sum"
|
|
92
|
+
if isinstance(node, Quantify):
|
|
93
|
+
return render_quantifier(node, reads, plan)
|
|
94
|
+
if isinstance(node, Merge):
|
|
95
|
+
return f"{name}<br/>merge"
|
|
96
|
+
return name
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def safe(text: str) -> str:
|
|
100
|
+
"""Mermaid takes the label between its shape markers, so anything that closes one has to go.
|
|
101
|
+
Labels are quoted, and quoted text may hold parentheses, which a guard needs to keep its
|
|
102
|
+
meaning."""
|
|
103
|
+
for character in '"[]{}|;\n':
|
|
104
|
+
text = text.replace(character, " ")
|
|
105
|
+
return " ".join(text.split())
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def mermaid(plan: Plan, closed: bool = False) -> str:
|
|
109
|
+
"""The plan as a Mermaid flowchart: boxes for nodes, arrows for what feeds what, a box of
|
|
110
|
+
its own for each condition, and a unit's graph drawn inside a box of its own, or, closed,
|
|
111
|
+
as one box, so a map of many units reads as a map."""
|
|
112
|
+
lines = ["flowchart TD"]
|
|
113
|
+
styled: dict[str, list[str]] = {}
|
|
114
|
+
draw(plan, "", lines, styled, closed)
|
|
115
|
+
for kind, members in styled.items():
|
|
116
|
+
if members:
|
|
117
|
+
lines.append(f" classDef {kind} {STYLES[kind]}")
|
|
118
|
+
lines.append(f" class {','.join(members)} {kind}")
|
|
119
|
+
return "\n".join(lines)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def guard(condition, reads: dict[str, Read], plan: Plan) -> str:
|
|
123
|
+
"""A condition as its box says it. In rules a bare name tests that the node answered, which
|
|
124
|
+
reads fine after `when`; alone in a box it needs the verb."""
|
|
125
|
+
# A test over a list's items already reads as a condition.
|
|
126
|
+
if isinstance(condition, Present) and not isinstance(plan.nodes.get(condition.node), Quantify):
|
|
127
|
+
return f"{name_of(condition.node, reads, plan)} is present"
|
|
128
|
+
return render_condition(condition, reads, plan)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def draw(
|
|
132
|
+
plan: Plan,
|
|
133
|
+
prefix: str,
|
|
134
|
+
lines: list[str],
|
|
135
|
+
styled: dict[str, list[str]],
|
|
136
|
+
closed: bool = False,
|
|
137
|
+
exit: tuple[str, str] | None = None,
|
|
138
|
+
) -> None:
|
|
139
|
+
"""One plan's nodes, edges and outputs, with ids under `prefix` so a unit's graph and the
|
|
140
|
+
graph around it never share an id. `exit` names the output that is a unit's answer and the
|
|
141
|
+
id of the unit's exit node, which stands in for that output's circle."""
|
|
142
|
+
reads = {name: node for name, node in plan.nodes.items() if isinstance(node, Read)}
|
|
143
|
+
# Mermaid joins boxes that share an id, and a node may be named like an output or anything
|
|
144
|
+
# else, so ids are positions and the names go in the labels.
|
|
145
|
+
ids = {name: f"{prefix}n{index}" for index, name in enumerate(plan.nodes)}
|
|
146
|
+
conditions: list[str] = []
|
|
147
|
+
# What a unit's gate and input attach to: the unit's whole box, since the condition decides
|
|
148
|
+
# whether any of it runs.
|
|
149
|
+
whole: dict[str, str] = {}
|
|
150
|
+
for name, node in plan.nodes.items():
|
|
151
|
+
if isinstance(node, Graph):
|
|
152
|
+
if closed:
|
|
153
|
+
# The double-edged box is the usual sign for a step that is a program of its
|
|
154
|
+
# own, which a closed unit is.
|
|
155
|
+
lines.append(f' {ids[name]}[["{safe(name)}<br/>a graph of its own"]]')
|
|
156
|
+
whole[name] = ids[name]
|
|
157
|
+
else:
|
|
158
|
+
whole[name] = f"{ids[name]}_unit"
|
|
159
|
+
# The gate's arrow lands on the frame's edge, which is easy to miss, so the
|
|
160
|
+
# frame's title says when the unit runs as well; on one line, since Mermaid
|
|
161
|
+
# gives a frame's title no room for a second.
|
|
162
|
+
title = f"{safe(name)}: a graph of its own"
|
|
163
|
+
if node.when is not None:
|
|
164
|
+
title += f", when {safe(guard(node.when, reads, plan))}"
|
|
165
|
+
lines.append(f' subgraph {whole[name]}["{title}"]')
|
|
166
|
+
answers = f"{safe(name)}<br/>answers with its {node.output}"
|
|
167
|
+
lines.append(f' {ids[name]}(["{answers}"])')
|
|
168
|
+
draw(node.plan, f"{ids[name]}_", lines, styled, closed, (node.output, ids[name]))
|
|
169
|
+
lines.append(" end")
|
|
170
|
+
if isinstance(node.input, NodeInput):
|
|
171
|
+
lines.append(f" {ids[node.input.node]} --> {whole[name]}")
|
|
172
|
+
continue
|
|
173
|
+
open_shape, close_shape = SHAPES[type(node)]
|
|
174
|
+
lines.append(
|
|
175
|
+
f' {ids[name]}{open_shape}"{safe(label(name, node, reads, plan))}"{close_shape}'
|
|
176
|
+
)
|
|
177
|
+
for name, node in plan.nodes.items():
|
|
178
|
+
if isinstance(node, Coalesce):
|
|
179
|
+
# Tried in order, so the order is the point: a judge last is asked only when every
|
|
180
|
+
# rule ahead of it declined.
|
|
181
|
+
for rank, source in enumerate(node.sources, start=1):
|
|
182
|
+
lines.append(f' {ids[source]} -->|"{rank}"| {ids[name]}')
|
|
183
|
+
continue
|
|
184
|
+
for source in sorted(feeds(node)):
|
|
185
|
+
lines.append(f" {ids[source]} --> {ids[name]}")
|
|
186
|
+
# A rule is often a bare value with a condition on it, so the guard has no data edge to
|
|
187
|
+
# ride on. It is a box of its own, fed by each value it reads, with a dashed arrow to
|
|
188
|
+
# what it gates: written on one arrow, a condition over two values looked as if it
|
|
189
|
+
# belonged to one of them.
|
|
190
|
+
if node.when is not None:
|
|
191
|
+
sources = sorted(tested(node.when))
|
|
192
|
+
box = f"{prefix}c{len(conditions)}"
|
|
193
|
+
conditions.append(box)
|
|
194
|
+
lines.append(f' {box}["{safe(guard(node.when, reads, plan))}"]')
|
|
195
|
+
lines += [f" {ids[source]} --> {box}" for source in sources]
|
|
196
|
+
lines.append(f" {box} -.-> {whole.get(name, ids[name])}")
|
|
197
|
+
for index, (output, source) in enumerate(plan.outputs.items()):
|
|
198
|
+
# A unit's answer leaves through its exit node; every other output is a circle.
|
|
199
|
+
if exit is not None and output == exit[0]:
|
|
200
|
+
target = exit[1]
|
|
201
|
+
else:
|
|
202
|
+
target = f"{prefix}o{index}"
|
|
203
|
+
lines.append(f' {target}(("{safe(output)}"))')
|
|
204
|
+
# Thick, so what the graph answers stands apart from what feeds what.
|
|
205
|
+
if isinstance(source, NodeInput):
|
|
206
|
+
lines.append(f" {ids[source.node]} ==> {target}")
|
|
207
|
+
for kind, types in KINDS.items():
|
|
208
|
+
styled.setdefault(kind, []).extend(
|
|
209
|
+
ids[name] for name, node in plan.nodes.items() if isinstance(node, types)
|
|
210
|
+
)
|
|
211
|
+
styled.setdefault("condition", []).extend(conditions)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def feeds(node) -> set[str]:
|
|
215
|
+
"""The nodes whose values this one consumes, ignoring the ones its guard mentions."""
|
|
216
|
+
if isinstance(node, (Call, Merge, Extract)):
|
|
217
|
+
return {source.node for source in node.inputs.values() if isinstance(source, NodeInput)}
|
|
218
|
+
if isinstance(node, Read) and node.source is not None:
|
|
219
|
+
return {node.source}
|
|
220
|
+
if isinstance(node, (Coalesce, Vote)):
|
|
221
|
+
return set(node.sources)
|
|
222
|
+
if isinstance(node, (Switch, Threshold, Confidence, Quantify)):
|
|
223
|
+
return {node.source}
|
|
224
|
+
if isinstance(node, Sum):
|
|
225
|
+
return {term.node for term in node.terms}
|
|
226
|
+
return set()
|