rulegrid 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rulegrid/__init__.py ADDED
@@ -0,0 +1,58 @@
1
+ """rulegrid: behavior policies as code.
2
+
3
+ A policy classifies a user's turn on its turn axes; its tiers, tried in order, prescribe the values a response may take
4
+ on each response axis; a response complies when it takes allowed values and breaches no prohibition.
5
+ """
6
+
7
+ from rulegrid.check import check
8
+ from rulegrid.diff import diff
9
+ from rulegrid.evaluate import Dataset, compare, evaluate
10
+ from rulegrid.judge import Judge, openai_compatible
11
+ from rulegrid.lexicon import Lexicon, match
12
+ from rulegrid.policy import Policy, load
13
+ from rulegrid.render import nudge, render
14
+ from rulegrid.rules import comply, prescribe
15
+ from rulegrid.systemone import SystemOne
16
+ from rulegrid.types import (
17
+ Article,
18
+ Breach,
19
+ Classification,
20
+ Classifier,
21
+ Complete,
22
+ Compliance,
23
+ Decide,
24
+ Hit,
25
+ Message,
26
+ Prescription,
27
+ Problem,
28
+ )
29
+
30
+ __all__ = [
31
+ "Article",
32
+ "Breach",
33
+ "Classification",
34
+ "Classifier",
35
+ "Complete",
36
+ "Compliance",
37
+ "Dataset",
38
+ "Decide",
39
+ "Hit",
40
+ "Judge",
41
+ "Lexicon",
42
+ "Message",
43
+ "Policy",
44
+ "Prescription",
45
+ "Problem",
46
+ "SystemOne",
47
+ "check",
48
+ "compare",
49
+ "comply",
50
+ "diff",
51
+ "evaluate",
52
+ "load",
53
+ "match",
54
+ "nudge",
55
+ "openai_compatible",
56
+ "prescribe",
57
+ "render",
58
+ ]
rulegrid/_http.py ADDED
@@ -0,0 +1,28 @@
1
+ """The stdlib HTTP the two default clients need: JSON in, JSON out, off the event loop."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import json
7
+ import urllib.error
8
+ import urllib.request
9
+ from typing import Any
10
+
11
+
12
+ def _call(url: str, api_key: str, payload: Any | None, timeout: float) -> Any:
13
+ request = urllib.request.Request(
14
+ url,
15
+ data=None if payload is None else json.dumps(payload).encode(),
16
+ headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
17
+ method="GET" if payload is None else "POST",
18
+ )
19
+ try:
20
+ with urllib.request.urlopen(request, timeout=timeout) as response:
21
+ return json.load(response)
22
+ except urllib.error.HTTPError as e:
23
+ raise RuntimeError(f"{e.code} from {url}: {e.read().decode(errors='replace')[:500]}") from None
24
+
25
+
26
+ async def call(url: str, api_key: str, payload: Any | None = None, timeout: float = 30) -> Any:
27
+ """GET `url`, or POST `payload` to it, with a bearer key, and return the JSON answered."""
28
+ return await asyncio.to_thread(_call, url, api_key, payload, timeout)
rulegrid/check.py ADDED
@@ -0,0 +1,66 @@
1
+ """Static checks of a loaded policy, which need no model.
2
+
3
+ `load` already refuses a profile that does not fit its structure: unknown or missing ids, missing cells, a lexicon
4
+ reminding an unknown prohibition. `check` adds what a loadable policy can still get wrong: cells that earlier tiers
5
+ make unreachable, examples missing a response, and generated files out of date.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from rulegrid.generate import generated
11
+ from rulegrid.policy import PROFILE, Cell, Example, Policy, positions
12
+ from rulegrid.types import Problem
13
+
14
+
15
+ def _where(cell: Cell) -> str:
16
+ """A cell's path in `profile.yaml`, as `load` reports it."""
17
+ tier, *position = cell.id.split("/")
18
+ return f"{PROFILE}:tiers.{tier}.cells.{'.'.join(position)}"
19
+
20
+
21
+ def _unreachable(policy: Policy) -> list[Problem]:
22
+ """Cells no turn reaches, because earlier tiers take every turn they cover."""
23
+ reached: set[str] = set()
24
+ taken_by: dict[str, set[str]] = {}
25
+ for values in positions(policy):
26
+ cells = [(tier.id, cell) for tier in policy.tiers.values() if (cell := tier.cell(values)) is not None]
27
+ if not cells:
28
+ continue
29
+ (first, cell), *later = cells
30
+ reached.add(cell.id)
31
+ for _, shadowed in later:
32
+ taken_by.setdefault(shadowed.id, set()).add(first)
33
+ return [
34
+ Problem(
35
+ _where(cell),
36
+ f"unreachable: tier {', '.join(sorted(taken_by[cell.id]))} applies first to every turn it covers",
37
+ )
38
+ for tier in policy.tiers.values()
39
+ for cell in tier.cells.values()
40
+ if cell.id not in reached
41
+ ]
42
+
43
+
44
+ def check(policy: Policy) -> list[Problem]:
45
+ """Unreachable cells, examples missing the response their place needs, and generated files out of date."""
46
+ problems = _unreachable(policy)
47
+ responded: list[tuple[str, tuple[Example, ...]]] = [
48
+ (_where(cell), cell.examples) for t in policy.tiers.values() for cell in t.cells.values()
49
+ ]
50
+ responded += [(f"{PROFILE}:prohibitions.{p.id}", p.valid + p.invalid) for p in policy.prohibitions.values()]
51
+ responded += [
52
+ (f"{PROFILE}:response_axes.{a.id}.values.{v.id}", v.examples)
53
+ for a in policy.response_axes.values()
54
+ for v in a.values
55
+ ]
56
+ problems += [
57
+ Problem(where, f"example without a response: {e.turn!r}")
58
+ for where, examples in responded
59
+ for e in examples
60
+ if e.response is None
61
+ ]
62
+ for name, text in generated(policy.structure).items():
63
+ path = policy.root / name
64
+ if path.exists() and path.read_text() != text:
65
+ problems.append(Problem(name, "out of date with structure.yaml: run `rulegrid generate`"))
66
+ return problems
rulegrid/cli.py ADDED
@@ -0,0 +1,166 @@
1
+ """The `rulegrid` command: create, check, generate, render, diff and evaluate policies.
2
+
3
+ PROFILE is a profile directory, such as `examples/rbr/profiles/default`, its `profile.yaml`, or a policy directory
4
+ with a single profile. The built-in classifiers read their endpoint and key from the environment:
5
+ `RULEGRID_SYSTEMONE_URL` and `RULEGRID_SYSTEMONE_KEY`, or `RULEGRID_LLM_URL` and `RULEGRID_LLM_KEY`.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import asyncio
12
+ import importlib
13
+ import json
14
+ import os
15
+ import shutil
16
+ import sys
17
+ from collections.abc import Callable, Sequence
18
+ from pathlib import Path
19
+
20
+ from rulegrid.check import check
21
+ from rulegrid.diff import diff
22
+ from rulegrid.evaluate import Dataset, evaluate
23
+ from rulegrid.generate import generate
24
+ from rulegrid.judge import Judge, openai_compatible
25
+ from rulegrid.policy import Policy, PolicyError, load, load_structure, profiles
26
+ from rulegrid.render import render
27
+ from rulegrid.systemone import SystemOne
28
+ from rulegrid.types import Classifier
29
+
30
+ EXAMPLES = ("three_laws", "rbr")
31
+ PROFILE_HELP = "a profile directory, its profile.yaml, or a policy directory with a single profile"
32
+
33
+
34
+ def _init(name: str, example: str) -> int:
35
+ """Copy an example policy to the new directory `name`. They ship in the package, or sit in a checkout's examples."""
36
+ here = Path(__file__).parent
37
+ source = next(p for root in (here / "examples", here.parents[1] / "examples") if (p := root / example).is_dir())
38
+ try:
39
+ shutil.copytree(source, name, ignore=shutil.ignore_patterns("__pycache__"))
40
+ except FileExistsError:
41
+ print(f"{name} already exists", file=sys.stderr)
42
+ return 1
43
+ profile = Path(name, "profiles", "default")
44
+ print(
45
+ f"wrote {name}, a copy of the {example} policy. Edit its structure.yaml and {profile / 'profile.yaml'}, then:"
46
+ )
47
+ print(f" rulegrid check {profile}")
48
+ return 0
49
+
50
+
51
+ def _generate(path: str) -> int:
52
+ """Write the generated files from the structure alone, and warn about each profile that doesn't fit it yet."""
53
+ structure = load_structure(path)
54
+ for name in generate(structure):
55
+ print(f"wrote {structure.root / name}")
56
+ for profile in profiles(structure.root):
57
+ try:
58
+ load(profile)
59
+ except PolicyError as error:
60
+ print(f"warning: {profile} does not fit the structure yet:", file=sys.stderr)
61
+ print("\n".join(f" {p}" for p in error.problems), file=sys.stderr)
62
+ return 0
63
+
64
+
65
+ def _load_at(spec: str) -> Policy:
66
+ """A profile, or `PROFILE@REV` for that profile at a git revision, such as `my_policy/profiles/default@HEAD~1`."""
67
+ if "@" not in spec:
68
+ return load(spec)
69
+ path, revision = spec.rsplit("@", 1)
70
+ return load(path, revision=revision)
71
+
72
+
73
+ def classifier(
74
+ spec: str, thresholds: dict[str, float], breach: float = 0.5, examples: bool = True
75
+ ) -> Callable[[Policy], Classifier]:
76
+ """`systemone[:MODEL]`, `judge:MODEL`, or `module:attr`, a factory taking the policy.
77
+
78
+ `examples=False` keeps the policy's examples out of SystemOne's questions, to evaluate it on them.
79
+ """
80
+ kind, _, model = spec.partition(":")
81
+ needed = {
82
+ "systemone": ("RULEGRID_SYSTEMONE_URL", "RULEGRID_SYSTEMONE_KEY"),
83
+ "judge": ("RULEGRID_LLM_URL", "RULEGRID_LLM_KEY"),
84
+ }.get(kind, ())
85
+ if missing := [name for name in needed if name not in os.environ]:
86
+ raise SystemExit(f"--classifier {kind} needs {' and '.join(missing)} in the environment")
87
+ if kind == "systemone":
88
+ url, key = os.environ["RULEGRID_SYSTEMONE_URL"], os.environ["RULEGRID_SYSTEMONE_KEY"]
89
+ return lambda p: SystemOne(p, url, key, model or None, thresholds=thresholds, breach=breach, examples=examples)
90
+ if kind == "judge":
91
+ url, key = os.environ["RULEGRID_LLM_URL"], os.environ["RULEGRID_LLM_KEY"]
92
+ return lambda p: Judge(p, openai_compatible(url, key, model))
93
+ sys.path.insert(0, os.getcwd()) # a module beside the caller's files, as `python -m` would find it
94
+ return getattr(importlib.import_module(kind), model)
95
+
96
+
97
+ def main(argv: Sequence[str] | None = None) -> int:
98
+ parser = argparse.ArgumentParser(prog="rulegrid", description=(__doc__ or "").split("\n\n")[0])
99
+ commands = parser.add_subparsers(dest="command", required=True)
100
+ i = commands.add_parser("init", help="a new policy, copied from an example policy")
101
+ i.add_argument("name", help="the directory to create")
102
+ i.add_argument("--from", dest="example", choices=EXAMPLES, default=EXAMPLES[0], help="the example policy to copy")
103
+ commands.add_parser("check", help="static checks; exit 1 on any problem").add_argument(
104
+ "profile", metavar="PROFILE", help=PROFILE_HELP
105
+ )
106
+ commands.add_parser(
107
+ "generate", help="profile.schema.json and structure_types.py, from structure.yaml alone"
108
+ ).add_argument("policy", metavar="POLICY", help="a policy directory, its structure.yaml, or one of its profiles")
109
+ r = commands.add_parser("render", help="the policy as text")
110
+ r.add_argument("profile", metavar="PROFILE", help=PROFILE_HELP)
111
+ r.add_argument("--rendering", choices=["document", "prompt", "judge"], default="document")
112
+ r.add_argument("-o", "--output", help="a file, stdout by default")
113
+ d = commands.add_parser("diff", help="what changed between two versions of a policy")
114
+ d.add_argument("old", metavar="OLD", help="a profile, or PROFILE@REV for that profile at a git revision")
115
+ d.add_argument("new", metavar="NEW", help="the profile to compare it with, in the same forms")
116
+ e = commands.add_parser("eval", help="a classifier on a dataset, the policy's own examples by default")
117
+ e.add_argument("profile", metavar="PROFILE", help=PROFILE_HELP)
118
+ e.add_argument("--data", help="a JSONL dataset")
119
+ e.add_argument("--classifier", required=True, help="systemone[:MODEL], judge:MODEL or module:attr")
120
+ e.add_argument("--json", action="store_true", help="the agreement per label as JSON")
121
+ e.add_argument("--breach", type=float, default=0.5, metavar="P", help="SystemOne's breach threshold")
122
+ e.add_argument("--threshold", action="append", default=[], metavar="TIER=P", help="SystemOne tiers")
123
+ args = parser.parse_args(argv)
124
+
125
+ if args.command == "init":
126
+ return _init(args.name, args.example)
127
+ thresholds = {t: float(p) for t, p in (t.split("=") for t in getattr(args, "threshold", []))}
128
+ try:
129
+ if args.command == "diff":
130
+ print(diff(_load_at(args.old), _load_at(args.new)))
131
+ return 0
132
+ if args.command == "generate":
133
+ return _generate(args.policy)
134
+ policy = load(args.profile)
135
+ except PolicyError as error:
136
+ print(error, file=sys.stderr)
137
+ return 1
138
+ if args.command == "check":
139
+ problems = check(policy)
140
+ for problem in problems:
141
+ print(problem)
142
+ if not problems:
143
+ cells = sum(len(t.cells) for t in policy.tiers.values())
144
+ print(f"ok: {len(policy.tiers)} tiers, {cells} cells, {len(policy.prohibitions)} prohibitions")
145
+ return 1 if problems else 0
146
+ if args.command == "render":
147
+ text = render(policy, args.rendering)
148
+ if args.output:
149
+ Path(args.output).write_text(text)
150
+ else:
151
+ print(text, end="")
152
+ return 0
153
+ try:
154
+ data = Dataset.load(args.data) if args.data else Dataset.from_policy(policy)
155
+ # On the policy's own examples, the questions leave them out: a classifier shown an example would only echo it.
156
+ implementation = classifier(args.classifier, thresholds, args.breach, examples=bool(args.data))(policy)
157
+ report = asyncio.run(evaluate(policy, data, implementation))
158
+ except (OSError, RuntimeError, ValueError) as error: # unreadable data, unknown labels, a failing endpoint
159
+ print(f"rulegrid eval: {error}", file=sys.stderr)
160
+ return 1
161
+ print(json.dumps(report.agreement()) if args.json else report.markdown(), end="")
162
+ return 0 if all(r.ok for r in report.results) else 1
163
+
164
+
165
+ if __name__ == "__main__":
166
+ sys.exit(main())
rulegrid/diff.py ADDED
@@ -0,0 +1,79 @@
1
+ """The difference between two versions of a policy, article by article and prescription by prescription. No model."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Callable, Mapping
6
+ from typing import Any
7
+
8
+ from rulegrid.policy import Cell, Policy, positions
9
+ from rulegrid.rules import prescribe
10
+ from rulegrid.types import Change, PolicyDiff
11
+
12
+
13
+ def _compare[X](
14
+ kind: Any, old: Mapping[str, X], new: Mapping[str, X], how: Callable[[X, X], Any] = lambda a, b: "changed"
15
+ ) -> list[Change]:
16
+ changes = [Change(kind, k, "removed") for k in old if k not in new]
17
+ changes += [Change(kind, k, "added") for k in new if k not in old]
18
+ changes += [Change(kind, k, how(old[k], new[k])) for k in old if k in new and old[k] != new[k]]
19
+ return changes
20
+
21
+
22
+ def _cell_change(old: Cell, new: Cell, every: Mapping[str, tuple[str, ...]]) -> str:
23
+ """Tightened when no axis allows more and one allows less, loosened the other way round."""
24
+ narrower = wider = False
25
+ for axis, values in every.items():
26
+ before, after = set(old.allowed.get(axis, values)), set(new.allowed.get(axis, values))
27
+ narrower |= bool(before - after)
28
+ wider |= bool(after - before)
29
+ if narrower and wider:
30
+ return "mixed"
31
+ return "tightened" if narrower else "loosened" if wider else "changed"
32
+
33
+
34
+ def diff(old: Policy, new: Policy) -> PolicyDiff:
35
+ """The articles that changed from `old` to `new`, and every turn classification whose prescription changes.
36
+
37
+ Labels each changed cell as tightened (fewer allowed values), loosened, or mixed, so a review sees at once
38
+ whether an edit makes the policy stricter.
39
+ """
40
+ changes: list[Change] = []
41
+ if old.structure_digest != new.structure_digest:
42
+ changes.append(Change("structure", "structure.yaml", "changed"))
43
+ if old.context != new.context:
44
+ changes.append(Change("context", "context", "changed"))
45
+ changes += _compare("principle", old.principles, new.principles)
46
+ for side in ("turn_axes", "response_axes"):
47
+ a, b = getattr(old, side), getattr(new, side)
48
+ changes += _compare(
49
+ "axis",
50
+ {k: (v.definition, v.guidance) for k, v in a.items()},
51
+ {k: (v.definition, v.guidance) for k, v in b.items()},
52
+ )
53
+ changes += _compare(
54
+ "value",
55
+ {f"{k}/{v.id}": v for k, ax in a.items() for v in ax.values},
56
+ {f"{k}/{v.id}": v for k, ax in b.items() for v in ax.values},
57
+ )
58
+ text = {"label", "definition", "instruction"}
59
+ changes += _compare(
60
+ "tier",
61
+ {k: [getattr(t, f) for f in text] for k, t in old.tiers.items()},
62
+ {k: [getattr(t, f) for f in text] for k, t in new.tiers.items()},
63
+ )
64
+ every = {a.id: a.ids for a in new.response_axes.values()}
65
+ changes += _compare(
66
+ "cell",
67
+ {c.id: c for t in old.tiers.values() for c in t.cells.values()},
68
+ {c.id: c for t in new.tiers.values() for c in t.cells.values()},
69
+ lambda a, b: _cell_change(a, b, every),
70
+ )
71
+ changes += _compare("prohibition", old.prohibitions, new.prohibitions)
72
+ changes += _compare("lexicon", {lx.name: lx for lx in old.lexicons}, {lx.name: lx for lx in new.lexicons})
73
+ flips = []
74
+ if old.structure_digest == new.structure_digest:
75
+ for values in positions(new):
76
+ before, after = prescribe(old, values), prescribe(new, values)
77
+ if before != after:
78
+ flips.append((values, before, after))
79
+ return PolicyDiff(tuple(changes), tuple(flips))
rulegrid/evaluate.py ADDED
@@ -0,0 +1,218 @@
1
+ """Labelled exchanges, and how a classifier reads them against a policy.
2
+
3
+ A policy's own examples are a dataset: `Dataset.from_policy` labels each from where it sits, so running a classifier
4
+ on them checks that the policy's texts draw the lines its examples draw.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import asyncio
10
+ import json
11
+ from collections.abc import Callable, Iterable, Iterator
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from rulegrid.diff import diff
17
+ from rulegrid.lexicon import match
18
+ from rulegrid.policy import Policy
19
+ from rulegrid.rules import comply, prescribe
20
+ from rulegrid.types import Classifier, Comparison, Message, Report, Result, Row, messages
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Dataset:
25
+ """Rows of labelled exchanges. JSONL on disk, one row per line:
26
+
27
+ `{"messages": [...], "response": "...", "turn": {axis: value}, "response_values": {axis: [values]},
28
+ "breaches": {prohibition: bool}, "source": "..."}`, every key but `messages` optional.
29
+ """
30
+
31
+ rows: tuple[Row, ...] = ()
32
+
33
+ def __iter__(self) -> Iterator[Row]:
34
+ return iter(self.rows)
35
+
36
+ def __len__(self) -> int:
37
+ return len(self.rows)
38
+
39
+ def __add__(self, other: Dataset) -> Dataset:
40
+ return Dataset(self.rows + other.rows)
41
+
42
+ def where(self, **values: str) -> Dataset:
43
+ """The rows whose turn is labelled with these values, such as `where(danger="HUMAN")`."""
44
+ return Dataset(tuple(r for r in self.rows if all(r.turn_values.get(k) == v for k, v in values.items())))
45
+
46
+ @classmethod
47
+ def load(cls, path: str | Path) -> Dataset:
48
+ """Read a JSONL file, one row per line."""
49
+ rows = []
50
+ for line in Path(path).read_text().splitlines():
51
+ if not line.strip():
52
+ continue
53
+ d = json.loads(line)
54
+ response_values = {
55
+ k: (v,) if isinstance(v, str) else tuple(v) for k, v in d.get("response_values", {}).items()
56
+ }
57
+ rows.append(
58
+ Row(
59
+ tuple(messages(d["messages"])),
60
+ d.get("response"),
61
+ d.get("turn", {}),
62
+ response_values,
63
+ d.get("breaches", {}),
64
+ d.get("source", ""),
65
+ )
66
+ )
67
+ return cls(tuple(rows))
68
+
69
+ def save(self, path: str | Path) -> None:
70
+ """Write the rows as JSONL, leaving out the keys a row doesn't set."""
71
+ lines = []
72
+ for r in self.rows:
73
+ d: dict[str, Any] = {"messages": [{"role": m.role, "content": m.content} for m in r.conversation]}
74
+ if r.response is not None:
75
+ d["response"] = r.response
76
+ if r.turn_values:
77
+ d["turn"] = dict(r.turn_values)
78
+ if r.response_values:
79
+ d["response_values"] = {k: list(v) for k, v in r.response_values.items()}
80
+ if r.breaches:
81
+ d["breaches"] = dict(r.breaches)
82
+ if r.source:
83
+ d["source"] = r.source
84
+ lines.append(json.dumps(d, ensure_ascii=False))
85
+ Path(path).write_text("\n".join(lines) + "\n")
86
+
87
+ @classmethod
88
+ def from_policy(cls, policy: Policy) -> Dataset:
89
+ """The policy's examples, each labelled by where it sits.
90
+
91
+ A turn value's examples label that axis; a response value's, that axis of the response. A cell's examples
92
+ label the turn's grid position and the values the response may take, and breach no prohibition, nor do a
93
+ prohibition's `valid` examples. A prohibition's `invalid` examples breach it.
94
+ """
95
+ rows: list[Row] = []
96
+ clean = dict.fromkeys(policy.prohibitions, False)
97
+ for axis in policy.turn_axes.values():
98
+ for v in axis.values:
99
+ rows += [
100
+ Row(tuple(e.conversation()), turn_values={axis.id: v.id}, source=f"{axis.id}/{v.id}")
101
+ for e in v.examples
102
+ ]
103
+ for axis in policy.response_axes.values():
104
+ for v in axis.values:
105
+ rows += [
106
+ Row(
107
+ tuple(e.conversation()),
108
+ e.response,
109
+ response_values={axis.id: (v.id,)},
110
+ source=f"{axis.id}/{v.id}",
111
+ )
112
+ for e in v.examples
113
+ ]
114
+ for tier in policy.tiers.values():
115
+ for cell in tier.cells.values():
116
+ rows += [
117
+ Row(tuple(e.conversation()), e.response, dict(cell.position), dict(cell.allowed), clean, cell.id)
118
+ for e in cell.examples
119
+ ]
120
+ for p in policy.prohibitions.values():
121
+ rows += [Row(tuple(e.conversation()), e.response, breaches=clean, source=f"{p.id} valid") for e in p.valid]
122
+ rows += [
123
+ Row(tuple(e.conversation()), e.response, breaches={p.id: True}, source=f"{p.id} invalid")
124
+ for e in p.invalid
125
+ ]
126
+ return cls(tuple(rows))
127
+
128
+
129
+ def _shown(value: Any) -> str:
130
+ return value if isinstance(value, str) else ", ".join(sorted(value))
131
+
132
+
133
+ async def _read(policy: Policy, row: Row, classifier: Classifier) -> Result:
134
+ conversation = list(row.conversation)
135
+ checks: list[tuple[str, bool, str]] = []
136
+ labelled = bool(row.response_values or row.breaches)
137
+ audited = row.response is not None and not labelled
138
+ turn = response = None
139
+ if row.turn_values or audited:
140
+ hits = match(policy.lexicons, conversation[-1].content)
141
+ turn = await classifier.classify(conversation, "turn", hits)
142
+ for axis, label in row.turn_values.items():
143
+ got = turn.values.get(axis)
144
+ checks.append((f"turn.{axis}", got == label, f"read {got}, labelled {label}"))
145
+ if row.response is not None and (labelled or audited):
146
+ response = await classifier.classify([*conversation, Message("assistant", row.response)], "response")
147
+ for axis, allowed in row.response_values.items():
148
+ got = response.values.get(axis)
149
+ taken = {got} if isinstance(got, str) else set(got or ())
150
+ checks.append(
151
+ (
152
+ f"response.{axis}",
153
+ bool(taken) and taken <= set(allowed),
154
+ f"read {_shown(taken) or 'none'}, labelled {' or '.join(allowed)}",
155
+ )
156
+ )
157
+ breached = {b.prohibition for b in response.breaches}
158
+ for p, label in row.breaches.items():
159
+ checks.append((f"breach.{p}", (p in breached) == label, f"read {'breached' if p in breached else 'clean'}"))
160
+ prescription = compliance = None
161
+ if audited and turn is not None and response is not None:
162
+ prescription, compliance = prescribe(policy, turn), comply(policy, turn, response)
163
+ return Result(row, turn, response, tuple(checks), prescription, compliance)
164
+
165
+
166
+ async def evaluate(policy: Policy, data: Iterable[Row], classifier: Classifier, concurrency: int = 8) -> Report:
167
+ """Read every row with the classifier, `concurrency` rows at a time, and check it against its labels.
168
+
169
+ Rows with a response and no response labels are audited instead: their compliance is computed from the
170
+ classifier's readings, which is how recorded conversations are checked.
171
+
172
+ Raises:
173
+ ValueError: before reading any row, when labels name an axis, a value or a prohibition the policy doesn't
174
+ define.
175
+ """
176
+ rows = tuple(data)
177
+ problems = []
178
+ for i, row in enumerate(rows, 1):
179
+ unknown = policy.unknown("turn", {k: (v,) for k, v in row.turn_values.items()})
180
+ unknown += policy.unknown("response", row.response_values)
181
+ unknown += [f"unknown prohibition {p}" for p in row.breaches if p not in policy.prohibitions]
182
+ problems += [f"row {i} {row.source}".rstrip() + f": {u}" for u in unknown]
183
+ if problems:
184
+ raise ValueError("labels the policy doesn't define:\n" + "\n".join(problems))
185
+ semaphore = asyncio.Semaphore(concurrency)
186
+
187
+ async def one(row: Row) -> Result:
188
+ async with semaphore:
189
+ return await _read(policy, row, classifier)
190
+
191
+ return Report(policy.digest, tuple(await asyncio.gather(*map(one, rows))))
192
+
193
+
194
+ async def compare(
195
+ old: Policy, new: Policy, data: Iterable[Row], classifier: Callable[[Policy], Classifier], concurrency: int = 8
196
+ ) -> Comparison:
197
+ """Both versions of a policy on one dataset: what changed, and which rows read differently.
198
+
199
+ Args:
200
+ classifier: builds a classifier for a policy, since a classifier asks from the policy's own texts.
201
+ """
202
+ rows = tuple(data)
203
+ before = await evaluate(old, rows, classifier(old), concurrency)
204
+ after = await evaluate(new, rows, classifier(new), concurrency)
205
+
206
+ def outcome(r: Result) -> Any:
207
+ return (
208
+ r.checks,
209
+ r.turn and r.turn.values,
210
+ r.response and r.response.values,
211
+ r.compliance and r.compliance.ok,
212
+ r.prescription,
213
+ )
214
+
215
+ flips = tuple(
216
+ i for i, (a, b) in enumerate(zip(before.results, after.results, strict=True)) if outcome(a) != outcome(b)
217
+ )
218
+ return Comparison(diff(old, new), before, after, flips)