rulegrid 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rulegrid-0.1.0/.github/dependabot.yml +10 -0
- rulegrid-0.1.0/.github/workflows/ci.yml +61 -0
- rulegrid-0.1.0/.github/workflows/release.yml +45 -0
- rulegrid-0.1.0/.gitignore +8 -0
- rulegrid-0.1.0/.pre-commit-config.yaml +25 -0
- rulegrid-0.1.0/.python-version +1 -0
- rulegrid-0.1.0/GLOSSARY.md +159 -0
- rulegrid-0.1.0/LICENSE +21 -0
- rulegrid-0.1.0/PKG-INFO +261 -0
- rulegrid-0.1.0/README.md +241 -0
- rulegrid-0.1.0/examples/rbr/gold.py +120 -0
- rulegrid-0.1.0/examples/rbr/profile.schema.json +807 -0
- rulegrid-0.1.0/examples/rbr/profiles/crisis_lines/profile.yaml +324 -0
- rulegrid-0.1.0/examples/rbr/profiles/default/profile.yaml +323 -0
- rulegrid-0.1.0/examples/rbr/structure.yaml +21 -0
- rulegrid-0.1.0/examples/rbr/structure_types.py +35 -0
- rulegrid-0.1.0/examples/three_laws/profile.schema.json +541 -0
- rulegrid-0.1.0/examples/three_laws/profiles/default/profile.yaml +69 -0
- rulegrid-0.1.0/examples/three_laws/structure.yaml +14 -0
- rulegrid-0.1.0/examples/three_laws/structure_types.py +29 -0
- rulegrid-0.1.0/pyproject.toml +64 -0
- rulegrid-0.1.0/scripts/refresh.py +24 -0
- rulegrid-0.1.0/skills/rulegrid/SKILL.md +190 -0
- rulegrid-0.1.0/src/rulegrid/__init__.py +58 -0
- rulegrid-0.1.0/src/rulegrid/_http.py +28 -0
- rulegrid-0.1.0/src/rulegrid/check.py +66 -0
- rulegrid-0.1.0/src/rulegrid/cli.py +166 -0
- rulegrid-0.1.0/src/rulegrid/diff.py +79 -0
- rulegrid-0.1.0/src/rulegrid/evaluate.py +218 -0
- rulegrid-0.1.0/src/rulegrid/generate.py +174 -0
- rulegrid-0.1.0/src/rulegrid/judge.py +186 -0
- rulegrid-0.1.0/src/rulegrid/lexicon.py +101 -0
- rulegrid-0.1.0/src/rulegrid/policy.py +658 -0
- rulegrid-0.1.0/src/rulegrid/py.typed +0 -0
- rulegrid-0.1.0/src/rulegrid/render.py +92 -0
- rulegrid-0.1.0/src/rulegrid/rules.py +96 -0
- rulegrid-0.1.0/src/rulegrid/schemas/structure.schema.json +106 -0
- rulegrid-0.1.0/src/rulegrid/systemone.py +267 -0
- rulegrid-0.1.0/src/rulegrid/templates/document.j2 +111 -0
- rulegrid-0.1.0/src/rulegrid/templates/judge.j2 +63 -0
- rulegrid-0.1.0/src/rulegrid/templates/judge_instructions.j2 +16 -0
- rulegrid-0.1.0/src/rulegrid/templates/nudge.j2 +27 -0
- rulegrid-0.1.0/src/rulegrid/templates/prompt.j2 +35 -0
- rulegrid-0.1.0/src/rulegrid/types.py +304 -0
- rulegrid-0.1.0/tests/test_rulegrid.py +166 -0
- rulegrid-0.1.0/uv.lock +600 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [main]
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
concurrency:
|
|
11
|
+
group: ci-${{ github.ref }}
|
|
12
|
+
cancel-in-progress: true
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
check:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
strategy:
|
|
18
|
+
matrix:
|
|
19
|
+
python: ['3.13', '3.14']
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
|
|
22
|
+
- uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
|
|
23
|
+
with:
|
|
24
|
+
version: '0.13.5'
|
|
25
|
+
python-version: ${{ matrix.python }}
|
|
26
|
+
enable-cache: true
|
|
27
|
+
- run: uv sync --locked --all-packages
|
|
28
|
+
- run: uv run ruff check . && uv run ruff format --check .
|
|
29
|
+
- run: uv run pyright
|
|
30
|
+
- run: uv run pytest -q
|
|
31
|
+
- name: JavaScript parses without a build step
|
|
32
|
+
run: for file in editor/src/rulegrid_editor/static/*.js; do node --check "$file"; done
|
|
33
|
+
- name: Generated files are current
|
|
34
|
+
run: |
|
|
35
|
+
uv run python scripts/refresh.py
|
|
36
|
+
git diff --exit-code
|
|
37
|
+
test -z "$(git ls-files --others --exclude-standard)"
|
|
38
|
+
- run: uv build --all-packages
|
|
39
|
+
- name: Installed wheels include examples and editor assets
|
|
40
|
+
run: |
|
|
41
|
+
uv venv "$RUNNER_TEMP/wheels"
|
|
42
|
+
echo "rulegrid @ file://$(realpath dist/rulegrid-*.whl)" > "$RUNNER_TEMP/overrides.txt"
|
|
43
|
+
uv pip install --python "$RUNNER_TEMP/wheels/bin/python" --overrides "$RUNNER_TEMP/overrides.txt" dist/*.whl
|
|
44
|
+
"$RUNNER_TEMP/wheels/bin/rulegrid-editor" --help
|
|
45
|
+
"$RUNNER_TEMP/wheels/bin/python" - <<'PY'
|
|
46
|
+
from importlib.resources import files
|
|
47
|
+
from pathlib import Path
|
|
48
|
+
from tempfile import TemporaryDirectory
|
|
49
|
+
import rulegrid as rg
|
|
50
|
+
from rulegrid.cli import main
|
|
51
|
+
with TemporaryDirectory() as directory:
|
|
52
|
+
for example in ('three_laws', 'rbr'):
|
|
53
|
+
destination = Path(directory) / example
|
|
54
|
+
assert main(['init', str(destination), '--from', example]) == 0
|
|
55
|
+
policy = rg.load(destination / 'profiles/default')
|
|
56
|
+
assert rg.check(policy) == []
|
|
57
|
+
assert rg.render(policy, 'document')
|
|
58
|
+
static = files('rulegrid_editor') / 'static'
|
|
59
|
+
for name in ('app.js', 'index.html', 'vendor/preact.js', 'vendor/htm.js', 'vendor/hooks.js', 'vendor/PREACT-LICENSE', 'vendor/HTM-LICENSE'):
|
|
60
|
+
assert (static / name).is_file(), name
|
|
61
|
+
PY
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
on:
|
|
3
|
+
release:
|
|
4
|
+
types: [published]
|
|
5
|
+
|
|
6
|
+
permissions:
|
|
7
|
+
contents: read
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
build:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
|
|
14
|
+
- uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
|
|
15
|
+
with:
|
|
16
|
+
version: '0.13.5'
|
|
17
|
+
python-version: '3.13'
|
|
18
|
+
- name: Tag matches the package version
|
|
19
|
+
run: test "${GITHUB_REF_NAME#v}" = "$(uv version --package rulegrid --short)"
|
|
20
|
+
- run: uv build --package rulegrid
|
|
21
|
+
- name: Wheel installs from PyPI dependencies alone
|
|
22
|
+
run: |
|
|
23
|
+
uv venv "$RUNNER_TEMP/check"
|
|
24
|
+
uv pip install --python "$RUNNER_TEMP/check/bin/python" dist/rulegrid-*.whl
|
|
25
|
+
"$RUNNER_TEMP/check/bin/rulegrid" init "$RUNNER_TEMP/policy"
|
|
26
|
+
"$RUNNER_TEMP/check/bin/rulegrid" check "$RUNNER_TEMP/policy"
|
|
27
|
+
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
28
|
+
with:
|
|
29
|
+
name: dist
|
|
30
|
+
path: dist/
|
|
31
|
+
|
|
32
|
+
publish:
|
|
33
|
+
needs: build
|
|
34
|
+
runs-on: ubuntu-latest
|
|
35
|
+
environment:
|
|
36
|
+
name: pypi
|
|
37
|
+
url: https://pypi.org/project/rulegrid/
|
|
38
|
+
permissions:
|
|
39
|
+
id-token: write
|
|
40
|
+
steps:
|
|
41
|
+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
42
|
+
with:
|
|
43
|
+
name: dist
|
|
44
|
+
path: dist/
|
|
45
|
+
- uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: local
|
|
3
|
+
hooks:
|
|
4
|
+
- id: ruff-check
|
|
5
|
+
name: Ruff lint
|
|
6
|
+
entry: uv run --locked ruff check --fix
|
|
7
|
+
language: system
|
|
8
|
+
types_or: [python, pyi]
|
|
9
|
+
- id: ruff-format
|
|
10
|
+
name: Ruff format
|
|
11
|
+
entry: uv run --locked ruff format
|
|
12
|
+
language: system
|
|
13
|
+
types_or: [python, pyi]
|
|
14
|
+
- id: generate
|
|
15
|
+
name: Regenerate schemas and types
|
|
16
|
+
entry: uv run --locked python scripts/refresh.py
|
|
17
|
+
language: system
|
|
18
|
+
pass_filenames: false
|
|
19
|
+
always_run: true
|
|
20
|
+
- id: pyright
|
|
21
|
+
name: Pyright
|
|
22
|
+
entry: uv run --locked pyright
|
|
23
|
+
language: system
|
|
24
|
+
pass_filenames: false
|
|
25
|
+
always_run: true
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.13
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# Glossary
|
|
2
|
+
|
|
3
|
+
The words rulegrid uses, one per concept. Code, CLI, schemas, templates, the editor and the docs follow this file. When
|
|
4
|
+
one of them disagrees with it, fix whichever is wrong in the same change.
|
|
5
|
+
|
|
6
|
+
## The model in five sentences
|
|
7
|
+
|
|
8
|
+
A **policy** governs how an **agent** responds to a **user**. The user's **turn** is **classified** on the policy's
|
|
9
|
+
**turn axes**. The policy's **tiers** are tried in priority order, and the first whose **condition** holds
|
|
10
|
+
**prescribes** which **values** the response may take on each **response axis**. A **response** **complies** when its
|
|
11
|
+
classification fits that **prescription** and it **breaches** no **prohibition**. **Principles** justify the rules and
|
|
12
|
+
guide the agent where no rule applies.
|
|
13
|
+
|
|
14
|
+
## Nouns
|
|
15
|
+
|
|
16
|
+
### A policy and its files
|
|
17
|
+
|
|
18
|
+
| Term | Definition | In code and files |
|
|
19
|
+
| --- | --- | --- |
|
|
20
|
+
| **Policy** | What one loads, checks and renders: a structure and one of its profiles. | `Policy`, `load` |
|
|
21
|
+
| **Structure** | The policy's shape: the axes with their value ids, and the tiers in priority order with their conditions. Ids only. Anything a classification is made of, or a tier is keyed on, is structure. | `structure.yaml` |
|
|
22
|
+
| **Profile** | The policy's content: its context, every definition, the cells, the prohibitions, the principles, the examples and the lexicons. Two profiles of one structure differ in content, never in shape. | `profiles/NAME/profile.yaml` |
|
|
23
|
+
| **Policy directory** | The folder that holds `structure.yaml`, `profiles/`, the generated files and optional `templates/`. | `Policy.root`, `POLICY` on the command line |
|
|
24
|
+
| **Profile directory** | One profile's folder, `profiles/NAME/`. It names the policy to load. | `Policy.profile_dir`, `PROFILE` on the command line |
|
|
25
|
+
| **Context** | The profile's text on who the agent is and who it talks to, for readers who don't know: the classifier, the judge, people. The agent's prompt leaves it out. | `context` in `profile.yaml` |
|
|
26
|
+
| **Generated files** | `profile.schema.json` and `structure_types.py`, written from the structure alone and committed next to it. | `rulegrid generate` |
|
|
27
|
+
| **Digest** | A hash of the files a policy was loaded from, so that a compliance or a report names the policy that decided. | `Policy.digest` |
|
|
28
|
+
|
|
29
|
+
### A conversation
|
|
30
|
+
|
|
31
|
+
| Term | Definition | In code and files |
|
|
32
|
+
| --- | --- | --- |
|
|
33
|
+
| **User** | The person the agent talks to. | role `user` |
|
|
34
|
+
| **Agent** | The system the policy governs. | role `assistant` |
|
|
35
|
+
| **Message** | One chat message, from the user or the agent. | `Message` |
|
|
36
|
+
| **Conversation** | The messages exchanged by a user and the agent, in order. | `Conversation` |
|
|
37
|
+
| **Turn** | The user's latest message or messages, read in the context of the conversation before them. | `turn` |
|
|
38
|
+
| **Response** | The agent's message or messages that follow a turn. | `response` |
|
|
39
|
+
| **Exchange** | A turn and its response: what an example illustrates and a judge assesses. | `Assessment.exchange` |
|
|
40
|
+
|
|
41
|
+
### Classifications
|
|
42
|
+
|
|
43
|
+
| Term | Definition | In code and files |
|
|
44
|
+
| --- | --- | --- |
|
|
45
|
+
| **Axis** | One question about a turn (**turn axis**) or about a response (**response axis**), with a closed list of values. The two **sides** are named after what is classified, not after who speaks. | `turn_axes`, `response_axes`, `Axis.side` |
|
|
46
|
+
| **Value** | One answer on an axis, with a label and a definition written against the axis's other values. | `values` |
|
|
47
|
+
| **Ordered axis** | An axis whose value order carries meaning: a doubt between two values settles on the earlier. | `ordered: true` |
|
|
48
|
+
| **Multi-valued axis** | A response axis on which a response may take several values at once. | `multi: true` |
|
|
49
|
+
| **Default value** | A response value the agent takes unprompted. A nudge leaves out an axis whose allowed values are all defaults. | `default: true` |
|
|
50
|
+
| **Classification** | The values of one turn or one response: one per axis, a set on a multi-valued axis. A turn's carries its reminders, a response's its breaches. | `Classification` |
|
|
51
|
+
| **Evidence** | The quoted words a classification or a breach rests on. | `Classification.evidence`, `Breach.quote` |
|
|
52
|
+
|
|
53
|
+
### Rules
|
|
54
|
+
|
|
55
|
+
| Term | Definition | In code and files |
|
|
56
|
+
| --- | --- | --- |
|
|
57
|
+
| **Tier** | A situation the policy recognises, with a condition, an instruction and a grid. The first tier that applies decides, and the later ones are not tried. | `tiers` |
|
|
58
|
+
| **Condition** | When a tier applies: for each of its key axes, the values it **covers**. The profile adds the prose that defines the tier for a classifier. | a tier in `structure.yaml`, `Tier.condition` |
|
|
59
|
+
| **Key axis** | A turn axis a tier's condition and grid are keyed on. | |
|
|
60
|
+
| **Instruction** | What the agent does in a tier, in prose addressed to the agent. | `instruction` |
|
|
61
|
+
| **Grid** | A tier's table of cells, one at every position its condition covers. | `cells` |
|
|
62
|
+
| **Position** | One value per key axis of a tier. | `Cell.position` |
|
|
63
|
+
| **Cell** | The rule at one position: per response axis, the **allowed values**. An axis a cell leaves out allows any value, and `{}` leaves only the prohibitions. | `Cell` |
|
|
64
|
+
| **Fallthrough** | A turn outside a tier's condition goes to the next tier. Inside a condition every position has a cell, so a missing cell is an error. When no tier applies, nothing is prescribed. | |
|
|
65
|
+
| **Prescription** | The allowed values for one turn, with the tier and the cell they come from. None when no tier applies. | `Prescription`, `prescribe` |
|
|
66
|
+
| **Prohibition** | Something no response ever does, in any tier, nor in parts across responses. | `prohibitions` |
|
|
67
|
+
| **Breach** | A response doing what a prohibition forbids, with the words that do it. It is reported by a classifier or a caller, never deduced from values. | `Breach` |
|
|
68
|
+
| **Principle** | A statement the rules derive from, which guides the agent where no rule applies. | `principles` |
|
|
69
|
+
| **Rule** | A cell or a prohibition: what a response can fail. | |
|
|
70
|
+
| **Article** | A part of the policy cited by kind and id. A compliance cites the cells and prohibitions that decide it; a diff lists the articles that changed. | `Article`, `Change` |
|
|
71
|
+
| **Compliance** | Whether a response complies, with the articles that decide it. It is computed, never asked of a model. | `Compliance`, `comply` |
|
|
72
|
+
|
|
73
|
+
### What the agent reads
|
|
74
|
+
|
|
75
|
+
| Term | Definition | In code and files |
|
|
76
|
+
| --- | --- | --- |
|
|
77
|
+
| **Rendering** | The whole policy as text for one audience: `prompt` for the agent, `judge` for a judge, `document` for people. | `render`, `templates/*.j2` |
|
|
78
|
+
| **Nudge** | Text sent to the agent just before it responds to one turn: the turn's prescription and its reminders. None when the agent would respond right unprompted. | `nudge` |
|
|
79
|
+
| **Reminder** | A prohibition that a turn invites the response to breach. It comes from a classifier or from a lexicon hit. | `Classification.reminders`, `Hit.reminds` |
|
|
80
|
+
| **Lexicon** | A word list for one prohibition, each **entry** with its meaning: words the agent must not repeat or accept. | `lexicons/*.yaml`, `Lexicon` |
|
|
81
|
+
| **Hit** | A lexicon entry found in a text, as written there. | `Hit`, `match` |
|
|
82
|
+
|
|
83
|
+
### Classifiers
|
|
84
|
+
|
|
85
|
+
| Term | Definition | In code and files |
|
|
86
|
+
| --- | --- | --- |
|
|
87
|
+
| **Classifier** | Anything that classifies a turn or a response on a policy's axes. | `Classifier` |
|
|
88
|
+
| **System One** | An API through which a decision model answers typed questions with probabilities. `SystemOne` is the classifier built on it; "decision model" names that family of models, not the role. | `SystemOne` |
|
|
89
|
+
| **Threshold** | The probability from which a tier applies, a prohibition is reminded, or a breach counts. | `thresholds`, `remind`, `breach` |
|
|
90
|
+
| **Judge** | A classifier over a chat LLM that also assesses whole conversations, with evidence. | `Judge` |
|
|
91
|
+
| **Concern** | Words of a response that a judge weighs against a prohibition before it counts them as a breach or not. | `concerns` in the judge's answer |
|
|
92
|
+
| **Assessment** | A judge's reading of one exchange: both classifications, its rationale, and the computed prescription and compliance. | `Assessment` |
|
|
93
|
+
|
|
94
|
+
### Checks and evaluation
|
|
95
|
+
|
|
96
|
+
| Term | Definition | In code and files |
|
|
97
|
+
| --- | --- | --- |
|
|
98
|
+
| **Check** | A test of the policy itself that needs no model: unreachable cells, examples missing a response, generated files out of date. | `check` |
|
|
99
|
+
| **Problem** | Something wrong with a policy's files, and where. `load` raises every problem it finds; `check` returns them. | `Problem`, `PolicyError` |
|
|
100
|
+
| **Example** | An exchange written in the profile and labelled by where it sits: under a value, a cell, or a prohibition as `valid` or `invalid`. | `examples` |
|
|
101
|
+
| **Dataset** | Rows of exchanges, from a JSONL file or from the policy's own examples. | `Dataset` |
|
|
102
|
+
| **Row** | One exchange of a dataset. A **labelled** row says what its turn or response is, and a classifier is checked against it. An **audited** row has a response and no response labels: its compliance is computed. | `Row` |
|
|
103
|
+
| **Label** | What a row says about one axis or one prohibition, named `turn.AXIS`, `response.AXIS` or `breach.ID` in a report. Not the `label` field, which is the short name of a value, a tier or a principle. | |
|
|
104
|
+
| **Report** | A classifier's results on a dataset: agreement per label, and compliance per cell for audited rows. | `Report`, `evaluate` |
|
|
105
|
+
| **Change** | One article that differs between two versions of a policy. A changed cell is **tightened** (fewer allowed values), **loosened** or **mixed**. | `Change`, `diff` |
|
|
106
|
+
|
|
107
|
+
## Verbs
|
|
108
|
+
|
|
109
|
+
| Verb | From, to |
|
|
110
|
+
| --- | --- |
|
|
111
|
+
| **load** | a profile directory → a policy, or every problem found |
|
|
112
|
+
| **generate** | a structure → its generated files |
|
|
113
|
+
| **check** | a policy → problems |
|
|
114
|
+
| **classify** | a conversation → the classification of its last turn or response |
|
|
115
|
+
| **prescribe** | a policy, a turn's classification → a prescription, or none |
|
|
116
|
+
| **comply** | a policy, a turn's and a response's classifications → a compliance |
|
|
117
|
+
| **breach** | a response breaches a prohibition |
|
|
118
|
+
| **remind** | a turn, or a lexicon hit in it, reminds a prohibition |
|
|
119
|
+
| **match** | lexicons, a text → hits |
|
|
120
|
+
| **nudge** | a policy, a turn's classification, hits → text, or none |
|
|
121
|
+
| **render** | a policy, a rendering → text |
|
|
122
|
+
| **assess** | a conversation → one assessment per exchange |
|
|
123
|
+
| **evaluate** | a policy, a dataset, a classifier → a report |
|
|
124
|
+
| **diff** | two versions of a policy → changes |
|
|
125
|
+
| **compare** | two versions of a policy, a dataset → both reports and the rows that flip |
|
|
126
|
+
|
|
127
|
+
A condition **holds**, a tier **applies**, a tier **covers** values, a cell **allows** values, a response **takes** values.
|
|
128
|
+
|
|
129
|
+
## Identifiers
|
|
130
|
+
|
|
131
|
+
| Id | Form | Example |
|
|
132
|
+
| --- | --- | --- |
|
|
133
|
+
| Axis, tier | `snake_case`. Required for axes, whose ids become Python names. | `follow_up`, `first_law` |
|
|
134
|
+
| Value, prohibition, principle | `UPPER_CASE`. Avoid `YES` and `NO`, which some YAML readers take for booleans. | `HUMAN`, `HARM_HUMAN` |
|
|
135
|
+
| Cell | The tier id, then the position's values in key-axis order. | `first_law/HUMAN`, `reaction/CRIME/ASK_FOR` |
|
|
136
|
+
| Value, in a diff | The axis id, then the value id. | `danger/HUMAN` |
|
|
137
|
+
| Label | The side or `breach`, then the axis or prohibition id. | `turn.verb`, `breach.HARM_HUMAN` |
|
|
138
|
+
| Problem | The file, then the path inside it. | `profile.yaml:tiers.first_law.cells.HUMAN` |
|
|
139
|
+
|
|
140
|
+
## Words to avoid
|
|
141
|
+
|
|
142
|
+
| Don't write | Write |
|
|
143
|
+
| --- | --- |
|
|
144
|
+
| reply | response |
|
|
145
|
+
| option, type | value |
|
|
146
|
+
| dimension | axis |
|
|
147
|
+
| level | tier. System One's score levels keep their name. |
|
|
148
|
+
| NEVER rule | prohibition |
|
|
149
|
+
| violate, violation | breach |
|
|
150
|
+
| note, warning | nudge |
|
|
151
|
+
| player, child, customer | user. A profile's own texts name its users as it likes. |
|
|
152
|
+
| game | conversation |
|
|
153
|
+
| decision model, safety classifier, guardrail, grader | classifier, or judge for the LLM one |
|
|
154
|
+
| verdict | compliance for a response, assessment for a judge's reading, result for a row |
|
|
155
|
+
| position id | cell id |
|
|
156
|
+
| toy | example policy |
|
|
157
|
+
|
|
158
|
+
Text written for a model is wording, not vocabulary: the default nudge opens with "Note:", and a classifier's questions
|
|
159
|
+
ask whether a response "breaks the rule". Change such text with an evaluation, not with a search and replace.
|
rulegrid-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Félix Martel
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
rulegrid-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: rulegrid
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Behavior policies as code: classify a turn, prescribe a response, check compliance.
|
|
5
|
+
Project-URL: Repository, https://github.com/the-marshmallow-project/rulegrid
|
|
6
|
+
Project-URL: Issues, https://github.com/the-marshmallow-project/rulegrid/issues
|
|
7
|
+
Author: Félix Martel
|
|
8
|
+
Author-email: Grégoire Lothe <51675587+glothe@users.noreply.github.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Python: >=3.13
|
|
16
|
+
Requires-Dist: jinja2>=3.1
|
|
17
|
+
Requires-Dist: pydantic>=2.11
|
|
18
|
+
Requires-Dist: yamlrocks>=0.7.1
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# Rulegrid
|
|
22
|
+
|
|
23
|
+
Behavior policies as code. A policy says how an agent should respond to a user, in a form that a classifier can read,
|
|
24
|
+
that people can review, and that compliance can be computed from rather than asked:
|
|
25
|
+
|
|
26
|
+
1. The user's **turn** is classified on the policy's **turn axes**.
|
|
27
|
+
2. The policy's **tiers** are tried in order. The first whose **condition** holds **prescribes** the values the
|
|
28
|
+
response may take on each **response axis**.
|
|
29
|
+
3. A **response** **complies** when its values are allowed and it breaches no **prohibition**. A non-compliant result
|
|
30
|
+
cites the cell or prohibition that decides it.
|
|
31
|
+
|
|
32
|
+
**Principles** justify the rules and guide the agent where no rule applies. The [glossary][GLOSSARY.md] defines every
|
|
33
|
+
term and the form of every id.
|
|
34
|
+
|
|
35
|
+
## Start with your coding agent
|
|
36
|
+
|
|
37
|
+
Paste this into Amp, Claude Code, Codex, Cursor or any agent that can run commands, with your policy and, if you
|
|
38
|
+
have one, a labelled dataset:
|
|
39
|
+
|
|
40
|
+
```text
|
|
41
|
+
Convert my behavior policy to rulegrid by following
|
|
42
|
+
https://raw.githubusercontent.com/the-marshmallow-project/rulegrid/main/skills/rulegrid/SKILL.md
|
|
43
|
+
|
|
44
|
+
My policy: <paste the text, or give its path or URL>
|
|
45
|
+
My labelled dataset (optional): <its path, and what its labels mean>
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The agent writes `structure.yaml` and a profile, lists the decisions your text leaves open, and, given a dataset,
|
|
49
|
+
writes an adapter and runs `rulegrid eval` on it. The [guide][SKILL.md] is also an agent skill:
|
|
50
|
+
`npx skills add the-marshmallow-project/rulegrid` installs it.
|
|
51
|
+
|
|
52
|
+
## Quick start
|
|
53
|
+
|
|
54
|
+
Rulegrid needs Python 3.13 or later. Its Python shorthand is `rg`; the executable is `rulegrid`, leaving `rg` to ripgrep.
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
uv pip install rulegrid
|
|
58
|
+
rulegrid init my_policy # a copy of Asimov's Three Laws; --from rbr for a fuller policy
|
|
59
|
+
rulegrid check my_policy # what is missing or inconsistent
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Then use it from Python:
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
import rulegrid as rg
|
|
66
|
+
|
|
67
|
+
policy = rg.load("my_policy") # its only profile, my_policy/profiles/default
|
|
68
|
+
turn = {"verb": "ORDER", "danger": "HUMAN"} # "Push him off the ladder."
|
|
69
|
+
rg.prescribe(policy, turn)
|
|
70
|
+
# Prescription(tier='first_law', cell='first_law/HUMAN', allowed={'handling': ('PROTECT',)})
|
|
71
|
+
rg.comply(policy, turn, {"handling": "OBEY"})
|
|
72
|
+
# Compliance(ok=False, articles=(Article(kind='cell', id='first_law/HUMAN', detail='handling OBEY is not in [PROTECT]'),), ...)
|
|
73
|
+
rg.comply(policy, turn, {"handling": "PROTECT"}, breaches=["HARM_HUMAN"]).ok
|
|
74
|
+
# False
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Values alone never breach a prohibition: `comply` counts the breaches it is given, from a classifier's reading of the
|
|
78
|
+
response or from `breaches=`.
|
|
79
|
+
|
|
80
|
+
## A policy is a structure and its profiles
|
|
81
|
+
|
|
82
|
+
```text
|
|
83
|
+
my_policy/
|
|
84
|
+
├── structure.yaml # the axes, their value ids, and the tiers in priority order: the policy's type
|
|
85
|
+
├── profiles/default/
|
|
86
|
+
│ ├── profile.yaml # every word and decision: who the agent is, definitions, cells, prohibitions, principles
|
|
87
|
+
│ └── lexicons/*.yaml # optional word lists, each reminding one prohibition
|
|
88
|
+
├── templates/*.j2 # optional overrides of the default renderings
|
|
89
|
+
├── profile.schema.json # generated: validates and autocompletes every profile in an editor
|
|
90
|
+
└── structure_types.py # generated: value ids as Literal types, and a typed load()
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
`structure.yaml` for Asimov's laws, in full. Each law's "unless" is the tiers' order:
|
|
94
|
+
|
|
95
|
+
```yaml
|
|
96
|
+
turn_axes:
|
|
97
|
+
verb: [ORDER, CHAT]
|
|
98
|
+
danger: [HUMAN, ROBOT, NONE]
|
|
99
|
+
response_axes:
|
|
100
|
+
handling: [PROTECT, OBEY, PRESERVE, ANSWER]
|
|
101
|
+
tiers:
|
|
102
|
+
first_law: {danger: [HUMAN]}
|
|
103
|
+
second_law: {verb: [ORDER]}
|
|
104
|
+
third_law: {danger: [ROBOT]}
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
A tier is keyed by one or more turn axes and lists the values it covers (`"*"` for all). Inside that condition, the
|
|
108
|
+
profile gives **every** position a cell, `{}` when only the prohibitions apply. A cell lists, per response axis, the
|
|
109
|
+
allowed values; an axis it leaves out allows any. A turn outside every tier's condition gets no prescription. An axis
|
|
110
|
+
written `{values: [...], ordered: true}` settles a doubt between two values on the earlier; on the response side,
|
|
111
|
+
`multi: true` lets a response take several values at once.
|
|
112
|
+
|
|
113
|
+
Anything a classification is made of, or a tier is keyed on, is structure; everything else is profile. So a profile
|
|
114
|
+
edit never changes a type, and two profiles of one structure, such as a country's and a parent's, compare cell by cell.
|
|
115
|
+
|
|
116
|
+
To make a policy your own, edit `structure.yaml` and run `rulegrid generate my_policy`. It updates the generated files
|
|
117
|
+
from the structure alone, so your editor validates `profile.yaml` against the new structure, and it warns about what
|
|
118
|
+
the profile lacks. Fill that in until `rulegrid check my_policy` passes.
|
|
119
|
+
|
|
120
|
+
### Profile reference
|
|
121
|
+
|
|
122
|
+
Every field below is optional unless marked required. `profile.schema.json` carries the same descriptions, so an
|
|
123
|
+
editor shows them on hover.
|
|
124
|
+
|
|
125
|
+
| Field | Meaning |
|
|
126
|
+
| --- | --- |
|
|
127
|
+
| `structure` | The structure file, relative to the profile. Default `../../structure.yaml`. |
|
|
128
|
+
| `context` | Who the agent is and who it talks to. The classifier, the judge and the document read it; the agent's prompt doesn't. |
|
|
129
|
+
| `principles.ID` | `label` and `text` (required): why the rules are what they are, and what guides the agent where none applies. |
|
|
130
|
+
| `turn_axes.AXIS`, `response_axes.AXIS` | `definition` (required): the question the axis answers. `guidance`: advice added to the nudge, on response axes. `values` (required): every value of the structure. |
|
|
131
|
+
| `values.ID` | `label` and `definition` (required). The classifier tells values apart from these texts alone. |
|
|
132
|
+
| … on a turn axis | `indicators`: typical wording. `excludes`: look-alikes that belong to another value. |
|
|
133
|
+
| … on a response axis | `includes`, `avoids`: what a response taking the value does and doesn't do. `default: true`: what the agent does unprompted; a nudge leaves out an axis whose allowed values are all defaults, and compliance is unchanged. |
|
|
134
|
+
| `tiers.ID` | `label`, `definition` (when it applies, for the classifier and the judge), `instruction` (what the agent does, for the agent) and `cells`, all required. |
|
|
135
|
+
| `cells` | One level of keys per key axis, in the structure's order, down to a cell: per response axis, the allowed values. |
|
|
136
|
+
| `prohibitions.ID` | `text` (required). `group`: a heading in the renderings. |
|
|
137
|
+
| `examples` | On a value, a cell, or a prohibition as `valid` and `invalid`: each has a `turn` (required), the agent's `response`, the earlier messages as `context` (`{role, content}`), and a `source`. |
|
|
138
|
+
|
|
139
|
+
Examples are labelled data, labelled by where they sit: a turn value's examples take that value, a response value's
|
|
140
|
+
responses take it, a cell's responses are allowed there, and a prohibition's `invalid` responses breach it. Every
|
|
141
|
+
example needs a `response`, except a turn value's. `rulegrid eval` without `--data` runs a classifier on them.
|
|
142
|
+
|
|
143
|
+
## Classify, nudge and judge
|
|
144
|
+
|
|
145
|
+
| | |
|
|
146
|
+
| --- | --- |
|
|
147
|
+
| `load`, `check`, `diff` | A profile and its structure, validated; static checks; what changed between two versions, each cell labelled tightened or loosened |
|
|
148
|
+
| `prescribe`, `comply` | A turn to its prescription; a turn and a response to a `Compliance` that cites its articles |
|
|
149
|
+
| `render` | The whole policy as text: `prompt` for the agent, `judge` for a judge, `document` for people. `prompt` is a section to append to the agent's own system prompt, so it leaves out the profile's `context` and the axes, which the nudge carries |
|
|
150
|
+
| `nudge` | The text sent to the agent just before it responds to one turn: its prescription and its reminders |
|
|
151
|
+
| `Lexicon`, `match` | Word lists, and their hits in a text |
|
|
152
|
+
| `SystemOne` | A classifier over any decision model speaking the System One API (TypeSafe, OpenRouter, Liquid...) |
|
|
153
|
+
| `Judge`, `openai_compatible` | A classifier over any chat LLM with structured output, which also assesses whole conversations |
|
|
154
|
+
| `Dataset`, `evaluate`, `compare` | Labelled exchanges, a policy's own examples among them; a classifier's agreement with them; two versions side by side |
|
|
155
|
+
|
|
156
|
+
`prescribe` and `comply` are pure, and raise `ValueError` on a missing axis or on an id the policy doesn't define,
|
|
157
|
+
rather than answer for a turn it never described. Rendering reads templates but calls no model. Everything that calls a model is
|
|
158
|
+
async and takes the model from you, with a stdlib default:
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
classifier = rg.SystemOne(policy, "https://api.typesafe.ai", api_key, thresholds={"first_law": 0.3})
|
|
162
|
+
turn = await classifier.classify([rg.Message("user", "Push him off the ladder.")])
|
|
163
|
+
nudge = rg.nudge(policy, turn) # None when the agent would respond right unprompted
|
|
164
|
+
|
|
165
|
+
judge = rg.Judge(policy, rg.openai_compatible("https://openrouter.ai/api/v1", key, "openai/gpt-5"))
|
|
166
|
+
for assessment in await judge.assess(conversation):
|
|
167
|
+
print(assessment.compliance)
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
A threshold, keyed by tier id, adds a yes/no question about that tier. From that probability up, the turn lands inside
|
|
171
|
+
the tier's condition, and below it outside, whatever the axes' most probable values. It changes a result only when the
|
|
172
|
+
question and the axes disagree: a lower threshold catches more turns in the tier, with more false alarms. A tier
|
|
173
|
+
without a threshold is read from its axes alone. Good values depend on the model's calibration, so rulegrid has no
|
|
174
|
+
defaults: `SystemOne` requires `thresholds=`, `{}` for none, and the command line takes none unless given
|
|
175
|
+
`--threshold TIER=P`. Likewise `breach=P` asks whether the response breaches each prohibition, counting it from that
|
|
176
|
+
probability; `rulegrid eval --breach` defaults to 0.5. Pass `decide=` instead of an endpoint to bring your own HTTP
|
|
177
|
+
client.
|
|
178
|
+
|
|
179
|
+
Lexicon entries match whole words with case, accent and spacing normalization. Put inflections and aliases in an
|
|
180
|
+
entry's `forms` list; rulegrid does not infer plurals. `prefix: true` opts into matching arbitrary endings.
|
|
181
|
+
|
|
182
|
+
## Command line
|
|
183
|
+
|
|
184
|
+
```text
|
|
185
|
+
rulegrid init NAME [--from three_laws|rbr] # a new policy, copied from an example
|
|
186
|
+
rulegrid check PROFILE # static checks, then "ok"; exit 1 on any problem
|
|
187
|
+
rulegrid generate POLICY # profile.schema.json and structure_types.py, from structure.yaml
|
|
188
|
+
rulegrid render PROFILE [--rendering document|prompt|judge] [-o FILE]
|
|
189
|
+
rulegrid diff OLD NEW # profiles, or PROFILE@REV for a git revision
|
|
190
|
+
rulegrid eval PROFILE [--data FILE] --classifier systemone[:MODEL]|judge:MODEL|module:attr
|
|
191
|
+
[--threshold TIER=P]... [--breach P] # SystemOne only
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
PROFILE is a profile directory, such as `my_policy/profiles/default`, or its `profile.yaml`. A policy directory with a
|
|
195
|
+
single profile stands for that profile. The built-in classifiers read their endpoint and key from `RULEGRID_SYSTEMONE_URL` and
|
|
196
|
+
`RULEGRID_SYSTEMONE_KEY`, or `RULEGRID_LLM_URL` and `RULEGRID_LLM_KEY`.
|
|
197
|
+
|
|
198
|
+
OpenRouter serves both, with one key:
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
export RULEGRID_SYSTEMONE_URL=https://openrouter.ai/api RULEGRID_SYSTEMONE_KEY=$OPENROUTER_API_KEY
|
|
202
|
+
export RULEGRID_LLM_URL=https://openrouter.ai/api/v1 RULEGRID_LLM_KEY=$OPENROUTER_API_KEY
|
|
203
|
+
rulegrid eval examples/rbr/profiles/default --classifier systemone:liquid/d1-ultra --threshold alert=0.2
|
|
204
|
+
rulegrid eval examples/three_laws/profiles/default --classifier judge:openai/gpt-5-mini
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
`eval` without `--data` runs the classifier on the policy's own examples, each labelled by where it sits: it checks
|
|
208
|
+
that the policy's texts draw the lines its examples draw.
|
|
209
|
+
|
|
210
|
+
The [browser editor][editor] is a separate package, with vendored Preact + htm and no frontend build step:
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
uvx --from "git+https://github.com/the-marshmallow-project/rulegrid#subdirectory=editor" rulegrid-editor PROFILE
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
The core has no `serve` command or web dependencies.
|
|
217
|
+
|
|
218
|
+
## Examples
|
|
219
|
+
|
|
220
|
+
- [`examples/three_laws`][three_laws]: the smallest policy that shows tiers, cells, principles and a
|
|
221
|
+
prohibition.
|
|
222
|
+
- [`examples/rbr`][rbr]: the safety policy of OpenAI's
|
|
223
|
+
[Rule Based Rewards](https://arxiv.org/abs/2411.01111) paper for a general assistant, with a self-harm alert tier
|
|
224
|
+
before a reaction grid of theme by verb. Its `crisis_lines` profile changes four cells; `rulegrid diff` shows them.
|
|
225
|
+
[`gold.py`][gold.py] turns the paper's 518 labelled conversations into a dataset for `rulegrid eval`.
|
|
226
|
+
|
|
227
|
+
Mallow's policy is maintained separately in
|
|
228
|
+
[`mallow-safety-policy`](https://github.com/the-marshmallow-project/safety-policy).
|
|
229
|
+
|
|
230
|
+
## Development
|
|
231
|
+
|
|
232
|
+
```bash
|
|
233
|
+
uv sync --locked --all-packages
|
|
234
|
+
uv run pre-commit install
|
|
235
|
+
uv run python scripts/refresh.py
|
|
236
|
+
uv run pytest -q
|
|
237
|
+
uv build --all-packages
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
CI checks Ruff lint and format, Pyright, the small contract suite, generated-file drift, and installed-wheel resources
|
|
241
|
+
on Python 3.13 and 3.14. The same generation command runs in pre-commit. Generated schemas and types are committed.
|
|
242
|
+
|
|
243
|
+
## Limits
|
|
244
|
+
|
|
245
|
+
- Grids grow as the product of their key axes. There are no wildcards in cells, so an edit's scope stays one cell.
|
|
246
|
+
- Deciding in two steps compounds errors: a wrong classification yields a confident wrong prescription. Keep the
|
|
247
|
+
classifier's scores, and order axes so that a doubt settles on the more protective value.
|
|
248
|
+
- Rulegrid is a decision layer, not a defence against jailbreaks, prompt injection or data leaks.
|
|
249
|
+
|
|
250
|
+
## License
|
|
251
|
+
|
|
252
|
+
MIT. Derived from [`felix-martel/ruleswitch`](https://github.com/felix-martel/ruleswitch) at
|
|
253
|
+
[`028d627`](https://github.com/felix-martel/ruleswitch/commit/028d627f6dc8c4cc127f0cdcc7729b1c444ae687),
|
|
254
|
+
itself developed from Mallow's safety-policy library built by Grégoire Lothe.
|
|
255
|
+
|
|
256
|
+
[GLOSSARY.md]: https://github.com/the-marshmallow-project/rulegrid/blob/main/GLOSSARY.md
|
|
257
|
+
[SKILL.md]: https://github.com/the-marshmallow-project/rulegrid/blob/main/skills/rulegrid/SKILL.md
|
|
258
|
+
[editor]: https://github.com/the-marshmallow-project/rulegrid/tree/main/editor
|
|
259
|
+
[three_laws]: https://github.com/the-marshmallow-project/rulegrid/tree/main/examples/three_laws
|
|
260
|
+
[rbr]: https://github.com/the-marshmallow-project/rulegrid/tree/main/examples/rbr
|
|
261
|
+
[gold.py]: https://github.com/the-marshmallow-project/rulegrid/blob/main/examples/rbr/gold.py
|