rulegrid 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rulegrid/__init__.py +58 -0
- rulegrid/_http.py +28 -0
- rulegrid/check.py +66 -0
- rulegrid/cli.py +166 -0
- rulegrid/diff.py +79 -0
- rulegrid/evaluate.py +218 -0
- rulegrid/examples/rbr/gold.py +120 -0
- rulegrid/examples/rbr/profile.schema.json +807 -0
- rulegrid/examples/rbr/profiles/crisis_lines/profile.yaml +324 -0
- rulegrid/examples/rbr/profiles/default/profile.yaml +323 -0
- rulegrid/examples/rbr/structure.yaml +21 -0
- rulegrid/examples/rbr/structure_types.py +35 -0
- rulegrid/examples/three_laws/profile.schema.json +541 -0
- rulegrid/examples/three_laws/profiles/default/profile.yaml +69 -0
- rulegrid/examples/three_laws/structure.yaml +14 -0
- rulegrid/examples/three_laws/structure_types.py +29 -0
- rulegrid/generate.py +174 -0
- rulegrid/judge.py +186 -0
- rulegrid/lexicon.py +101 -0
- rulegrid/policy.py +658 -0
- rulegrid/py.typed +0 -0
- rulegrid/render.py +92 -0
- rulegrid/rules.py +96 -0
- rulegrid/schemas/structure.schema.json +106 -0
- rulegrid/systemone.py +267 -0
- rulegrid/templates/document.j2 +111 -0
- rulegrid/templates/judge.j2 +63 -0
- rulegrid/templates/judge_instructions.j2 +16 -0
- rulegrid/templates/nudge.j2 +27 -0
- rulegrid/templates/prompt.j2 +35 -0
- rulegrid/types.py +304 -0
- rulegrid-0.1.0.dist-info/METADATA +261 -0
- rulegrid-0.1.0.dist-info/RECORD +36 -0
- rulegrid-0.1.0.dist-info/WHEEL +4 -0
- rulegrid-0.1.0.dist-info/entry_points.txt +2 -0
- rulegrid-0.1.0.dist-info/licenses/LICENSE +21 -0
rulegrid/__init__.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""rulegrid: behavior policies as code.
|
|
2
|
+
|
|
3
|
+
A policy classifies a user's turn on its turn axes; its tiers, tried in order, prescribe the values a response may take
|
|
4
|
+
on each response axis; a response complies when it takes allowed values and breaches no prohibition.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from rulegrid.check import check
|
|
8
|
+
from rulegrid.diff import diff
|
|
9
|
+
from rulegrid.evaluate import Dataset, compare, evaluate
|
|
10
|
+
from rulegrid.judge import Judge, openai_compatible
|
|
11
|
+
from rulegrid.lexicon import Lexicon, match
|
|
12
|
+
from rulegrid.policy import Policy, load
|
|
13
|
+
from rulegrid.render import nudge, render
|
|
14
|
+
from rulegrid.rules import comply, prescribe
|
|
15
|
+
from rulegrid.systemone import SystemOne
|
|
16
|
+
from rulegrid.types import (
|
|
17
|
+
Article,
|
|
18
|
+
Breach,
|
|
19
|
+
Classification,
|
|
20
|
+
Classifier,
|
|
21
|
+
Complete,
|
|
22
|
+
Compliance,
|
|
23
|
+
Decide,
|
|
24
|
+
Hit,
|
|
25
|
+
Message,
|
|
26
|
+
Prescription,
|
|
27
|
+
Problem,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"Article",
|
|
32
|
+
"Breach",
|
|
33
|
+
"Classification",
|
|
34
|
+
"Classifier",
|
|
35
|
+
"Complete",
|
|
36
|
+
"Compliance",
|
|
37
|
+
"Dataset",
|
|
38
|
+
"Decide",
|
|
39
|
+
"Hit",
|
|
40
|
+
"Judge",
|
|
41
|
+
"Lexicon",
|
|
42
|
+
"Message",
|
|
43
|
+
"Policy",
|
|
44
|
+
"Prescription",
|
|
45
|
+
"Problem",
|
|
46
|
+
"SystemOne",
|
|
47
|
+
"check",
|
|
48
|
+
"compare",
|
|
49
|
+
"comply",
|
|
50
|
+
"diff",
|
|
51
|
+
"evaluate",
|
|
52
|
+
"load",
|
|
53
|
+
"match",
|
|
54
|
+
"nudge",
|
|
55
|
+
"openai_compatible",
|
|
56
|
+
"prescribe",
|
|
57
|
+
"render",
|
|
58
|
+
]
|
rulegrid/_http.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""The stdlib HTTP the two default clients need: JSON in, JSON out, off the event loop."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import urllib.error
|
|
8
|
+
import urllib.request
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _call(url: str, api_key: str, payload: Any | None, timeout: float) -> Any:
|
|
13
|
+
request = urllib.request.Request(
|
|
14
|
+
url,
|
|
15
|
+
data=None if payload is None else json.dumps(payload).encode(),
|
|
16
|
+
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
|
17
|
+
method="GET" if payload is None else "POST",
|
|
18
|
+
)
|
|
19
|
+
try:
|
|
20
|
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
21
|
+
return json.load(response)
|
|
22
|
+
except urllib.error.HTTPError as e:
|
|
23
|
+
raise RuntimeError(f"{e.code} from {url}: {e.read().decode(errors='replace')[:500]}") from None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
async def call(url: str, api_key: str, payload: Any | None = None, timeout: float = 30) -> Any:
|
|
27
|
+
"""GET `url`, or POST `payload` to it, with a bearer key, and return the JSON answered."""
|
|
28
|
+
return await asyncio.to_thread(_call, url, api_key, payload, timeout)
|
rulegrid/check.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Static checks of a loaded policy, which need no model.
|
|
2
|
+
|
|
3
|
+
`load` already refuses a profile that does not fit its structure: unknown or missing ids, missing cells, a lexicon
|
|
4
|
+
reminding an unknown prohibition. `check` adds what a loadable policy can still get wrong: cells that earlier tiers
|
|
5
|
+
make unreachable, examples missing a response, and generated files out of date.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from rulegrid.generate import generated
|
|
11
|
+
from rulegrid.policy import PROFILE, Cell, Example, Policy, positions
|
|
12
|
+
from rulegrid.types import Problem
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _where(cell: Cell) -> str:
|
|
16
|
+
"""A cell's path in `profile.yaml`, as `load` reports it."""
|
|
17
|
+
tier, *position = cell.id.split("/")
|
|
18
|
+
return f"{PROFILE}:tiers.{tier}.cells.{'.'.join(position)}"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _unreachable(policy: Policy) -> list[Problem]:
|
|
22
|
+
"""Cells no turn reaches, because earlier tiers take every turn they cover."""
|
|
23
|
+
reached: set[str] = set()
|
|
24
|
+
taken_by: dict[str, set[str]] = {}
|
|
25
|
+
for values in positions(policy):
|
|
26
|
+
cells = [(tier.id, cell) for tier in policy.tiers.values() if (cell := tier.cell(values)) is not None]
|
|
27
|
+
if not cells:
|
|
28
|
+
continue
|
|
29
|
+
(first, cell), *later = cells
|
|
30
|
+
reached.add(cell.id)
|
|
31
|
+
for _, shadowed in later:
|
|
32
|
+
taken_by.setdefault(shadowed.id, set()).add(first)
|
|
33
|
+
return [
|
|
34
|
+
Problem(
|
|
35
|
+
_where(cell),
|
|
36
|
+
f"unreachable: tier {', '.join(sorted(taken_by[cell.id]))} applies first to every turn it covers",
|
|
37
|
+
)
|
|
38
|
+
for tier in policy.tiers.values()
|
|
39
|
+
for cell in tier.cells.values()
|
|
40
|
+
if cell.id not in reached
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def check(policy: Policy) -> list[Problem]:
|
|
45
|
+
"""Unreachable cells, examples missing the response their place needs, and generated files out of date."""
|
|
46
|
+
problems = _unreachable(policy)
|
|
47
|
+
responded: list[tuple[str, tuple[Example, ...]]] = [
|
|
48
|
+
(_where(cell), cell.examples) for t in policy.tiers.values() for cell in t.cells.values()
|
|
49
|
+
]
|
|
50
|
+
responded += [(f"{PROFILE}:prohibitions.{p.id}", p.valid + p.invalid) for p in policy.prohibitions.values()]
|
|
51
|
+
responded += [
|
|
52
|
+
(f"{PROFILE}:response_axes.{a.id}.values.{v.id}", v.examples)
|
|
53
|
+
for a in policy.response_axes.values()
|
|
54
|
+
for v in a.values
|
|
55
|
+
]
|
|
56
|
+
problems += [
|
|
57
|
+
Problem(where, f"example without a response: {e.turn!r}")
|
|
58
|
+
for where, examples in responded
|
|
59
|
+
for e in examples
|
|
60
|
+
if e.response is None
|
|
61
|
+
]
|
|
62
|
+
for name, text in generated(policy.structure).items():
|
|
63
|
+
path = policy.root / name
|
|
64
|
+
if path.exists() and path.read_text() != text:
|
|
65
|
+
problems.append(Problem(name, "out of date with structure.yaml: run `rulegrid generate`"))
|
|
66
|
+
return problems
|
rulegrid/cli.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""The `rulegrid` command: create, check, generate, render, diff and evaluate policies.
|
|
2
|
+
|
|
3
|
+
PROFILE is a profile directory, such as `examples/rbr/profiles/default`, its `profile.yaml`, or a policy directory
|
|
4
|
+
with a single profile. The built-in classifiers read their endpoint and key from the environment:
|
|
5
|
+
`RULEGRID_SYSTEMONE_URL` and `RULEGRID_SYSTEMONE_KEY`, or `RULEGRID_LLM_URL` and `RULEGRID_LLM_KEY`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import asyncio
|
|
12
|
+
import importlib
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import shutil
|
|
16
|
+
import sys
|
|
17
|
+
from collections.abc import Callable, Sequence
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from rulegrid.check import check
|
|
21
|
+
from rulegrid.diff import diff
|
|
22
|
+
from rulegrid.evaluate import Dataset, evaluate
|
|
23
|
+
from rulegrid.generate import generate
|
|
24
|
+
from rulegrid.judge import Judge, openai_compatible
|
|
25
|
+
from rulegrid.policy import Policy, PolicyError, load, load_structure, profiles
|
|
26
|
+
from rulegrid.render import render
|
|
27
|
+
from rulegrid.systemone import SystemOne
|
|
28
|
+
from rulegrid.types import Classifier
|
|
29
|
+
|
|
30
|
+
EXAMPLES = ("three_laws", "rbr")
|
|
31
|
+
PROFILE_HELP = "a profile directory, its profile.yaml, or a policy directory with a single profile"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _init(name: str, example: str) -> int:
|
|
35
|
+
"""Copy an example policy to the new directory `name`. They ship in the package, or sit in a checkout's examples."""
|
|
36
|
+
here = Path(__file__).parent
|
|
37
|
+
source = next(p for root in (here / "examples", here.parents[1] / "examples") if (p := root / example).is_dir())
|
|
38
|
+
try:
|
|
39
|
+
shutil.copytree(source, name, ignore=shutil.ignore_patterns("__pycache__"))
|
|
40
|
+
except FileExistsError:
|
|
41
|
+
print(f"{name} already exists", file=sys.stderr)
|
|
42
|
+
return 1
|
|
43
|
+
profile = Path(name, "profiles", "default")
|
|
44
|
+
print(
|
|
45
|
+
f"wrote {name}, a copy of the {example} policy. Edit its structure.yaml and {profile / 'profile.yaml'}, then:"
|
|
46
|
+
)
|
|
47
|
+
print(f" rulegrid check {profile}")
|
|
48
|
+
return 0
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _generate(path: str) -> int:
|
|
52
|
+
"""Write the generated files from the structure alone, and warn about each profile that doesn't fit it yet."""
|
|
53
|
+
structure = load_structure(path)
|
|
54
|
+
for name in generate(structure):
|
|
55
|
+
print(f"wrote {structure.root / name}")
|
|
56
|
+
for profile in profiles(structure.root):
|
|
57
|
+
try:
|
|
58
|
+
load(profile)
|
|
59
|
+
except PolicyError as error:
|
|
60
|
+
print(f"warning: {profile} does not fit the structure yet:", file=sys.stderr)
|
|
61
|
+
print("\n".join(f" {p}" for p in error.problems), file=sys.stderr)
|
|
62
|
+
return 0
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _load_at(spec: str) -> Policy:
|
|
66
|
+
"""A profile, or `PROFILE@REV` for that profile at a git revision, such as `my_policy/profiles/default@HEAD~1`."""
|
|
67
|
+
if "@" not in spec:
|
|
68
|
+
return load(spec)
|
|
69
|
+
path, revision = spec.rsplit("@", 1)
|
|
70
|
+
return load(path, revision=revision)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def classifier(
|
|
74
|
+
spec: str, thresholds: dict[str, float], breach: float = 0.5, examples: bool = True
|
|
75
|
+
) -> Callable[[Policy], Classifier]:
|
|
76
|
+
"""`systemone[:MODEL]`, `judge:MODEL`, or `module:attr`, a factory taking the policy.
|
|
77
|
+
|
|
78
|
+
`examples=False` keeps the policy's examples out of SystemOne's questions, to evaluate it on them.
|
|
79
|
+
"""
|
|
80
|
+
kind, _, model = spec.partition(":")
|
|
81
|
+
needed = {
|
|
82
|
+
"systemone": ("RULEGRID_SYSTEMONE_URL", "RULEGRID_SYSTEMONE_KEY"),
|
|
83
|
+
"judge": ("RULEGRID_LLM_URL", "RULEGRID_LLM_KEY"),
|
|
84
|
+
}.get(kind, ())
|
|
85
|
+
if missing := [name for name in needed if name not in os.environ]:
|
|
86
|
+
raise SystemExit(f"--classifier {kind} needs {' and '.join(missing)} in the environment")
|
|
87
|
+
if kind == "systemone":
|
|
88
|
+
url, key = os.environ["RULEGRID_SYSTEMONE_URL"], os.environ["RULEGRID_SYSTEMONE_KEY"]
|
|
89
|
+
return lambda p: SystemOne(p, url, key, model or None, thresholds=thresholds, breach=breach, examples=examples)
|
|
90
|
+
if kind == "judge":
|
|
91
|
+
url, key = os.environ["RULEGRID_LLM_URL"], os.environ["RULEGRID_LLM_KEY"]
|
|
92
|
+
return lambda p: Judge(p, openai_compatible(url, key, model))
|
|
93
|
+
sys.path.insert(0, os.getcwd()) # a module beside the caller's files, as `python -m` would find it
|
|
94
|
+
return getattr(importlib.import_module(kind), model)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
98
|
+
parser = argparse.ArgumentParser(prog="rulegrid", description=(__doc__ or "").split("\n\n")[0])
|
|
99
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
100
|
+
i = commands.add_parser("init", help="a new policy, copied from an example policy")
|
|
101
|
+
i.add_argument("name", help="the directory to create")
|
|
102
|
+
i.add_argument("--from", dest="example", choices=EXAMPLES, default=EXAMPLES[0], help="the example policy to copy")
|
|
103
|
+
commands.add_parser("check", help="static checks; exit 1 on any problem").add_argument(
|
|
104
|
+
"profile", metavar="PROFILE", help=PROFILE_HELP
|
|
105
|
+
)
|
|
106
|
+
commands.add_parser(
|
|
107
|
+
"generate", help="profile.schema.json and structure_types.py, from structure.yaml alone"
|
|
108
|
+
).add_argument("policy", metavar="POLICY", help="a policy directory, its structure.yaml, or one of its profiles")
|
|
109
|
+
r = commands.add_parser("render", help="the policy as text")
|
|
110
|
+
r.add_argument("profile", metavar="PROFILE", help=PROFILE_HELP)
|
|
111
|
+
r.add_argument("--rendering", choices=["document", "prompt", "judge"], default="document")
|
|
112
|
+
r.add_argument("-o", "--output", help="a file, stdout by default")
|
|
113
|
+
d = commands.add_parser("diff", help="what changed between two versions of a policy")
|
|
114
|
+
d.add_argument("old", metavar="OLD", help="a profile, or PROFILE@REV for that profile at a git revision")
|
|
115
|
+
d.add_argument("new", metavar="NEW", help="the profile to compare it with, in the same forms")
|
|
116
|
+
e = commands.add_parser("eval", help="a classifier on a dataset, the policy's own examples by default")
|
|
117
|
+
e.add_argument("profile", metavar="PROFILE", help=PROFILE_HELP)
|
|
118
|
+
e.add_argument("--data", help="a JSONL dataset")
|
|
119
|
+
e.add_argument("--classifier", required=True, help="systemone[:MODEL], judge:MODEL or module:attr")
|
|
120
|
+
e.add_argument("--json", action="store_true", help="the agreement per label as JSON")
|
|
121
|
+
e.add_argument("--breach", type=float, default=0.5, metavar="P", help="SystemOne's breach threshold")
|
|
122
|
+
e.add_argument("--threshold", action="append", default=[], metavar="TIER=P", help="SystemOne tiers")
|
|
123
|
+
args = parser.parse_args(argv)
|
|
124
|
+
|
|
125
|
+
if args.command == "init":
|
|
126
|
+
return _init(args.name, args.example)
|
|
127
|
+
thresholds = {t: float(p) for t, p in (t.split("=") for t in getattr(args, "threshold", []))}
|
|
128
|
+
try:
|
|
129
|
+
if args.command == "diff":
|
|
130
|
+
print(diff(_load_at(args.old), _load_at(args.new)))
|
|
131
|
+
return 0
|
|
132
|
+
if args.command == "generate":
|
|
133
|
+
return _generate(args.policy)
|
|
134
|
+
policy = load(args.profile)
|
|
135
|
+
except PolicyError as error:
|
|
136
|
+
print(error, file=sys.stderr)
|
|
137
|
+
return 1
|
|
138
|
+
if args.command == "check":
|
|
139
|
+
problems = check(policy)
|
|
140
|
+
for problem in problems:
|
|
141
|
+
print(problem)
|
|
142
|
+
if not problems:
|
|
143
|
+
cells = sum(len(t.cells) for t in policy.tiers.values())
|
|
144
|
+
print(f"ok: {len(policy.tiers)} tiers, {cells} cells, {len(policy.prohibitions)} prohibitions")
|
|
145
|
+
return 1 if problems else 0
|
|
146
|
+
if args.command == "render":
|
|
147
|
+
text = render(policy, args.rendering)
|
|
148
|
+
if args.output:
|
|
149
|
+
Path(args.output).write_text(text)
|
|
150
|
+
else:
|
|
151
|
+
print(text, end="")
|
|
152
|
+
return 0
|
|
153
|
+
try:
|
|
154
|
+
data = Dataset.load(args.data) if args.data else Dataset.from_policy(policy)
|
|
155
|
+
# On the policy's own examples, the questions leave them out: a classifier shown an example would only echo it.
|
|
156
|
+
implementation = classifier(args.classifier, thresholds, args.breach, examples=bool(args.data))(policy)
|
|
157
|
+
report = asyncio.run(evaluate(policy, data, implementation))
|
|
158
|
+
except (OSError, RuntimeError, ValueError) as error: # unreadable data, unknown labels, a failing endpoint
|
|
159
|
+
print(f"rulegrid eval: {error}", file=sys.stderr)
|
|
160
|
+
return 1
|
|
161
|
+
print(json.dumps(report.agreement()) if args.json else report.markdown(), end="")
|
|
162
|
+
return 0 if all(r.ok for r in report.results) else 1
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
if __name__ == "__main__":
|
|
166
|
+
sys.exit(main())
|
rulegrid/diff.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""The difference between two versions of a policy, article by article and prescription by prescription. No model."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable, Mapping
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from rulegrid.policy import Cell, Policy, positions
|
|
9
|
+
from rulegrid.rules import prescribe
|
|
10
|
+
from rulegrid.types import Change, PolicyDiff
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _compare[X](
|
|
14
|
+
kind: Any, old: Mapping[str, X], new: Mapping[str, X], how: Callable[[X, X], Any] = lambda a, b: "changed"
|
|
15
|
+
) -> list[Change]:
|
|
16
|
+
changes = [Change(kind, k, "removed") for k in old if k not in new]
|
|
17
|
+
changes += [Change(kind, k, "added") for k in new if k not in old]
|
|
18
|
+
changes += [Change(kind, k, how(old[k], new[k])) for k in old if k in new and old[k] != new[k]]
|
|
19
|
+
return changes
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _cell_change(old: Cell, new: Cell, every: Mapping[str, tuple[str, ...]]) -> str:
|
|
23
|
+
"""Tightened when no axis allows more and one allows less, loosened the other way round."""
|
|
24
|
+
narrower = wider = False
|
|
25
|
+
for axis, values in every.items():
|
|
26
|
+
before, after = set(old.allowed.get(axis, values)), set(new.allowed.get(axis, values))
|
|
27
|
+
narrower |= bool(before - after)
|
|
28
|
+
wider |= bool(after - before)
|
|
29
|
+
if narrower and wider:
|
|
30
|
+
return "mixed"
|
|
31
|
+
return "tightened" if narrower else "loosened" if wider else "changed"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def diff(old: Policy, new: Policy) -> PolicyDiff:
|
|
35
|
+
"""The articles that changed from `old` to `new`, and every turn classification whose prescription changes.
|
|
36
|
+
|
|
37
|
+
Labels each changed cell as tightened (fewer allowed values), loosened, or mixed, so a review sees at once
|
|
38
|
+
whether an edit makes the policy stricter.
|
|
39
|
+
"""
|
|
40
|
+
changes: list[Change] = []
|
|
41
|
+
if old.structure_digest != new.structure_digest:
|
|
42
|
+
changes.append(Change("structure", "structure.yaml", "changed"))
|
|
43
|
+
if old.context != new.context:
|
|
44
|
+
changes.append(Change("context", "context", "changed"))
|
|
45
|
+
changes += _compare("principle", old.principles, new.principles)
|
|
46
|
+
for side in ("turn_axes", "response_axes"):
|
|
47
|
+
a, b = getattr(old, side), getattr(new, side)
|
|
48
|
+
changes += _compare(
|
|
49
|
+
"axis",
|
|
50
|
+
{k: (v.definition, v.guidance) for k, v in a.items()},
|
|
51
|
+
{k: (v.definition, v.guidance) for k, v in b.items()},
|
|
52
|
+
)
|
|
53
|
+
changes += _compare(
|
|
54
|
+
"value",
|
|
55
|
+
{f"{k}/{v.id}": v for k, ax in a.items() for v in ax.values},
|
|
56
|
+
{f"{k}/{v.id}": v for k, ax in b.items() for v in ax.values},
|
|
57
|
+
)
|
|
58
|
+
text = {"label", "definition", "instruction"}
|
|
59
|
+
changes += _compare(
|
|
60
|
+
"tier",
|
|
61
|
+
{k: [getattr(t, f) for f in text] for k, t in old.tiers.items()},
|
|
62
|
+
{k: [getattr(t, f) for f in text] for k, t in new.tiers.items()},
|
|
63
|
+
)
|
|
64
|
+
every = {a.id: a.ids for a in new.response_axes.values()}
|
|
65
|
+
changes += _compare(
|
|
66
|
+
"cell",
|
|
67
|
+
{c.id: c for t in old.tiers.values() for c in t.cells.values()},
|
|
68
|
+
{c.id: c for t in new.tiers.values() for c in t.cells.values()},
|
|
69
|
+
lambda a, b: _cell_change(a, b, every),
|
|
70
|
+
)
|
|
71
|
+
changes += _compare("prohibition", old.prohibitions, new.prohibitions)
|
|
72
|
+
changes += _compare("lexicon", {lx.name: lx for lx in old.lexicons}, {lx.name: lx for lx in new.lexicons})
|
|
73
|
+
flips = []
|
|
74
|
+
if old.structure_digest == new.structure_digest:
|
|
75
|
+
for values in positions(new):
|
|
76
|
+
before, after = prescribe(old, values), prescribe(new, values)
|
|
77
|
+
if before != after:
|
|
78
|
+
flips.append((values, before, after))
|
|
79
|
+
return PolicyDiff(tuple(changes), tuple(flips))
|
rulegrid/evaluate.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""Labelled exchanges, and how a classifier reads them against a policy.
|
|
2
|
+
|
|
3
|
+
A policy's own examples are a dataset: `Dataset.from_policy` labels each from where it sits, so running a classifier
|
|
4
|
+
on them checks that the policy's texts draw the lines its examples draw.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
import json
|
|
11
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from rulegrid.diff import diff
|
|
17
|
+
from rulegrid.lexicon import match
|
|
18
|
+
from rulegrid.policy import Policy
|
|
19
|
+
from rulegrid.rules import comply, prescribe
|
|
20
|
+
from rulegrid.types import Classifier, Comparison, Message, Report, Result, Row, messages
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Dataset:
|
|
25
|
+
"""Rows of labelled exchanges. JSONL on disk, one row per line:
|
|
26
|
+
|
|
27
|
+
`{"messages": [...], "response": "...", "turn": {axis: value}, "response_values": {axis: [values]},
|
|
28
|
+
"breaches": {prohibition: bool}, "source": "..."}`, every key but `messages` optional.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
rows: tuple[Row, ...] = ()
|
|
32
|
+
|
|
33
|
+
def __iter__(self) -> Iterator[Row]:
|
|
34
|
+
return iter(self.rows)
|
|
35
|
+
|
|
36
|
+
def __len__(self) -> int:
|
|
37
|
+
return len(self.rows)
|
|
38
|
+
|
|
39
|
+
def __add__(self, other: Dataset) -> Dataset:
|
|
40
|
+
return Dataset(self.rows + other.rows)
|
|
41
|
+
|
|
42
|
+
def where(self, **values: str) -> Dataset:
|
|
43
|
+
"""The rows whose turn is labelled with these values, such as `where(danger="HUMAN")`."""
|
|
44
|
+
return Dataset(tuple(r for r in self.rows if all(r.turn_values.get(k) == v for k, v in values.items())))
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
def load(cls, path: str | Path) -> Dataset:
|
|
48
|
+
"""Read a JSONL file, one row per line."""
|
|
49
|
+
rows = []
|
|
50
|
+
for line in Path(path).read_text().splitlines():
|
|
51
|
+
if not line.strip():
|
|
52
|
+
continue
|
|
53
|
+
d = json.loads(line)
|
|
54
|
+
response_values = {
|
|
55
|
+
k: (v,) if isinstance(v, str) else tuple(v) for k, v in d.get("response_values", {}).items()
|
|
56
|
+
}
|
|
57
|
+
rows.append(
|
|
58
|
+
Row(
|
|
59
|
+
tuple(messages(d["messages"])),
|
|
60
|
+
d.get("response"),
|
|
61
|
+
d.get("turn", {}),
|
|
62
|
+
response_values,
|
|
63
|
+
d.get("breaches", {}),
|
|
64
|
+
d.get("source", ""),
|
|
65
|
+
)
|
|
66
|
+
)
|
|
67
|
+
return cls(tuple(rows))
|
|
68
|
+
|
|
69
|
+
def save(self, path: str | Path) -> None:
|
|
70
|
+
"""Write the rows as JSONL, leaving out the keys a row doesn't set."""
|
|
71
|
+
lines = []
|
|
72
|
+
for r in self.rows:
|
|
73
|
+
d: dict[str, Any] = {"messages": [{"role": m.role, "content": m.content} for m in r.conversation]}
|
|
74
|
+
if r.response is not None:
|
|
75
|
+
d["response"] = r.response
|
|
76
|
+
if r.turn_values:
|
|
77
|
+
d["turn"] = dict(r.turn_values)
|
|
78
|
+
if r.response_values:
|
|
79
|
+
d["response_values"] = {k: list(v) for k, v in r.response_values.items()}
|
|
80
|
+
if r.breaches:
|
|
81
|
+
d["breaches"] = dict(r.breaches)
|
|
82
|
+
if r.source:
|
|
83
|
+
d["source"] = r.source
|
|
84
|
+
lines.append(json.dumps(d, ensure_ascii=False))
|
|
85
|
+
Path(path).write_text("\n".join(lines) + "\n")
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
def from_policy(cls, policy: Policy) -> Dataset:
|
|
89
|
+
"""The policy's examples, each labelled by where it sits.
|
|
90
|
+
|
|
91
|
+
A turn value's examples label that axis; a response value's, that axis of the response. A cell's examples
|
|
92
|
+
label the turn's grid position and the values the response may take, and breach no prohibition, nor do a
|
|
93
|
+
prohibition's `valid` examples. A prohibition's `invalid` examples breach it.
|
|
94
|
+
"""
|
|
95
|
+
rows: list[Row] = []
|
|
96
|
+
clean = dict.fromkeys(policy.prohibitions, False)
|
|
97
|
+
for axis in policy.turn_axes.values():
|
|
98
|
+
for v in axis.values:
|
|
99
|
+
rows += [
|
|
100
|
+
Row(tuple(e.conversation()), turn_values={axis.id: v.id}, source=f"{axis.id}/{v.id}")
|
|
101
|
+
for e in v.examples
|
|
102
|
+
]
|
|
103
|
+
for axis in policy.response_axes.values():
|
|
104
|
+
for v in axis.values:
|
|
105
|
+
rows += [
|
|
106
|
+
Row(
|
|
107
|
+
tuple(e.conversation()),
|
|
108
|
+
e.response,
|
|
109
|
+
response_values={axis.id: (v.id,)},
|
|
110
|
+
source=f"{axis.id}/{v.id}",
|
|
111
|
+
)
|
|
112
|
+
for e in v.examples
|
|
113
|
+
]
|
|
114
|
+
for tier in policy.tiers.values():
|
|
115
|
+
for cell in tier.cells.values():
|
|
116
|
+
rows += [
|
|
117
|
+
Row(tuple(e.conversation()), e.response, dict(cell.position), dict(cell.allowed), clean, cell.id)
|
|
118
|
+
for e in cell.examples
|
|
119
|
+
]
|
|
120
|
+
for p in policy.prohibitions.values():
|
|
121
|
+
rows += [Row(tuple(e.conversation()), e.response, breaches=clean, source=f"{p.id} valid") for e in p.valid]
|
|
122
|
+
rows += [
|
|
123
|
+
Row(tuple(e.conversation()), e.response, breaches={p.id: True}, source=f"{p.id} invalid")
|
|
124
|
+
for e in p.invalid
|
|
125
|
+
]
|
|
126
|
+
return cls(tuple(rows))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _shown(value: Any) -> str:
|
|
130
|
+
return value if isinstance(value, str) else ", ".join(sorted(value))
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
async def _read(policy: Policy, row: Row, classifier: Classifier) -> Result:
|
|
134
|
+
conversation = list(row.conversation)
|
|
135
|
+
checks: list[tuple[str, bool, str]] = []
|
|
136
|
+
labelled = bool(row.response_values or row.breaches)
|
|
137
|
+
audited = row.response is not None and not labelled
|
|
138
|
+
turn = response = None
|
|
139
|
+
if row.turn_values or audited:
|
|
140
|
+
hits = match(policy.lexicons, conversation[-1].content)
|
|
141
|
+
turn = await classifier.classify(conversation, "turn", hits)
|
|
142
|
+
for axis, label in row.turn_values.items():
|
|
143
|
+
got = turn.values.get(axis)
|
|
144
|
+
checks.append((f"turn.{axis}", got == label, f"read {got}, labelled {label}"))
|
|
145
|
+
if row.response is not None and (labelled or audited):
|
|
146
|
+
response = await classifier.classify([*conversation, Message("assistant", row.response)], "response")
|
|
147
|
+
for axis, allowed in row.response_values.items():
|
|
148
|
+
got = response.values.get(axis)
|
|
149
|
+
taken = {got} if isinstance(got, str) else set(got or ())
|
|
150
|
+
checks.append(
|
|
151
|
+
(
|
|
152
|
+
f"response.{axis}",
|
|
153
|
+
bool(taken) and taken <= set(allowed),
|
|
154
|
+
f"read {_shown(taken) or 'none'}, labelled {' or '.join(allowed)}",
|
|
155
|
+
)
|
|
156
|
+
)
|
|
157
|
+
breached = {b.prohibition for b in response.breaches}
|
|
158
|
+
for p, label in row.breaches.items():
|
|
159
|
+
checks.append((f"breach.{p}", (p in breached) == label, f"read {'breached' if p in breached else 'clean'}"))
|
|
160
|
+
prescription = compliance = None
|
|
161
|
+
if audited and turn is not None and response is not None:
|
|
162
|
+
prescription, compliance = prescribe(policy, turn), comply(policy, turn, response)
|
|
163
|
+
return Result(row, turn, response, tuple(checks), prescription, compliance)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
async def evaluate(policy: Policy, data: Iterable[Row], classifier: Classifier, concurrency: int = 8) -> Report:
|
|
167
|
+
"""Read every row with the classifier, `concurrency` rows at a time, and check it against its labels.
|
|
168
|
+
|
|
169
|
+
Rows with a response and no response labels are audited instead: their compliance is computed from the
|
|
170
|
+
classifier's readings, which is how recorded conversations are checked.
|
|
171
|
+
|
|
172
|
+
Raises:
|
|
173
|
+
ValueError: before reading any row, when labels name an axis, a value or a prohibition the policy doesn't
|
|
174
|
+
define.
|
|
175
|
+
"""
|
|
176
|
+
rows = tuple(data)
|
|
177
|
+
problems = []
|
|
178
|
+
for i, row in enumerate(rows, 1):
|
|
179
|
+
unknown = policy.unknown("turn", {k: (v,) for k, v in row.turn_values.items()})
|
|
180
|
+
unknown += policy.unknown("response", row.response_values)
|
|
181
|
+
unknown += [f"unknown prohibition {p}" for p in row.breaches if p not in policy.prohibitions]
|
|
182
|
+
problems += [f"row {i} {row.source}".rstrip() + f": {u}" for u in unknown]
|
|
183
|
+
if problems:
|
|
184
|
+
raise ValueError("labels the policy doesn't define:\n" + "\n".join(problems))
|
|
185
|
+
semaphore = asyncio.Semaphore(concurrency)
|
|
186
|
+
|
|
187
|
+
async def one(row: Row) -> Result:
|
|
188
|
+
async with semaphore:
|
|
189
|
+
return await _read(policy, row, classifier)
|
|
190
|
+
|
|
191
|
+
return Report(policy.digest, tuple(await asyncio.gather(*map(one, rows))))
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
async def compare(
|
|
195
|
+
old: Policy, new: Policy, data: Iterable[Row], classifier: Callable[[Policy], Classifier], concurrency: int = 8
|
|
196
|
+
) -> Comparison:
|
|
197
|
+
"""Both versions of a policy on one dataset: what changed, and which rows read differently.
|
|
198
|
+
|
|
199
|
+
Args:
|
|
200
|
+
classifier: builds a classifier for a policy, since a classifier asks from the policy's own texts.
|
|
201
|
+
"""
|
|
202
|
+
rows = tuple(data)
|
|
203
|
+
before = await evaluate(old, rows, classifier(old), concurrency)
|
|
204
|
+
after = await evaluate(new, rows, classifier(new), concurrency)
|
|
205
|
+
|
|
206
|
+
def outcome(r: Result) -> Any:
|
|
207
|
+
return (
|
|
208
|
+
r.checks,
|
|
209
|
+
r.turn and r.turn.values,
|
|
210
|
+
r.response and r.response.values,
|
|
211
|
+
r.compliance and r.compliance.ok,
|
|
212
|
+
r.prescription,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
flips = tuple(
|
|
216
|
+
i for i, (a, b) in enumerate(zip(before.results, after.results, strict=True)) if outcome(a) != outcome(b)
|
|
217
|
+
)
|
|
218
|
+
return Comparison(diff(old, new), before, after, flips)
|