ridi-check 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ridi_check-0.3.2/LICENSE +21 -0
- ridi_check-0.3.2/MANIFEST.in +2 -0
- ridi_check-0.3.2/PKG-INFO +224 -0
- ridi_check-0.3.2/README.md +193 -0
- ridi_check-0.3.2/pyproject.toml +54 -0
- ridi_check-0.3.2/setup.cfg +4 -0
- ridi_check-0.3.2/src/ridi_check/__init__.py +3 -0
- ridi_check-0.3.2/src/ridi_check/__main__.py +6 -0
- ridi_check-0.3.2/src/ridi_check/adapters.py +257 -0
- ridi_check-0.3.2/src/ridi_check/analysis.py +190 -0
- ridi_check-0.3.2/src/ridi_check/cli.py +337 -0
- ridi_check-0.3.2/src/ridi_check/contract.py +222 -0
- ridi_check-0.3.2/src/ridi_check/demo/questions.csv +31 -0
- ridi_check-0.3.2/src/ridi_check/demo/ridi-check.json +11 -0
- ridi_check-0.3.2/src/ridi_check/demo/toy_assistant.py +154 -0
- ridi_check-0.3.2/src/ridi_check/gate.py +76 -0
- ridi_check-0.3.2/src/ridi_check/report.py +412 -0
- ridi_check-0.3.2/src/ridi_check/report_template.html +321 -0
- ridi_check-0.3.2/src/ridi_check/runner.py +501 -0
- ridi_check-0.3.2/src/ridi_check/scoring.py +21 -0
- ridi_check-0.3.2/src/ridi_check/starter/ASK_YOUR_AI.md +36 -0
- ridi_check-0.3.2/src/ridi_check/starter/my_assistant.py +37 -0
- ridi_check-0.3.2/src/ridi_check/starter/questions.csv +4 -0
- ridi_check-0.3.2/src/ridi_check/starter/results_template.csv +4 -0
- ridi_check-0.3.2/src/ridi_check/starter/ridi-check.json +10 -0
- ridi_check-0.3.2/src/ridi_check/strings.py +242 -0
- ridi_check-0.3.2/src/ridi_check/util.py +70 -0
- ridi_check-0.3.2/src/ridi_check.egg-info/PKG-INFO +224 -0
- ridi_check-0.3.2/src/ridi_check.egg-info/SOURCES.txt +31 -0
- ridi_check-0.3.2/src/ridi_check.egg-info/dependency_links.txt +1 -0
- ridi_check-0.3.2/src/ridi_check.egg-info/entry_points.txt +2 -0
- ridi_check-0.3.2/src/ridi_check.egg-info/requires.txt +3 -0
- ridi_check-0.3.2/src/ridi_check.egg-info/top_level.txt +1 -0
ridi_check-0.3.2/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ridi-check contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ridi-check
|
|
3
|
+
Version: 0.3.2
|
|
4
|
+
Summary: See what a model or prompt update changed in an LLM assistant's answers, and which changes hold on re-run.
|
|
5
|
+
Author: ridi-check contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/adeebnoor/ridi-check
|
|
8
|
+
Project-URL: Documentation, https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md
|
|
9
|
+
Project-URL: Changelog, https://github.com/adeebnoor/ridi-check/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Issues, https://github.com/adeebnoor/ridi-check/issues
|
|
11
|
+
Keywords: llm,evals,regression-testing,ai-agents,prompt-testing,rag,ci
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
23
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
24
|
+
Classifier: Topic :: Software Development :: Testing
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Provides-Extra: test
|
|
29
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# ridi-check
|
|
33
|
+
|
|
34
|
+
**See what a model or prompt update really changed in your assistant's answers, before you ship it.**
|
|
35
|
+
|
|
36
|
+
[](https://github.com/adeebnoor/ridi-check/actions/workflows/tests.yml)
|
|
37
|
+
[](https://github.com/adeebnoor/ridi-check/blob/main/LICENSE)
|
|
38
|
+
[](https://github.com/adeebnoor/ridi-check/blob/main/pyproject.toml)
|
|
39
|
+
|
|
40
|
+
A new model or an edited prompt can change what your assistant or agent answers without anyone noticing:
|
|
41
|
+
average accuracy barely moves while individual answers flip. ridi-check runs the old and the new version on
|
|
42
|
+
the same questions, more than once, and shows which answers changed and which of those changes hold on
|
|
43
|
+
re-run. You need no test set, no answer key and no model judge.
|
|
44
|
+
|
|
45
|
+

|
|
46
|
+
|
|
47
|
+
[What you will see](#what-you-will-see) · [Who it is for](#who-it-is-for) · [Start in one minute](#start-in-one-minute) ·
|
|
48
|
+
[Use it in CI](#use-it-in-ci) · [Connect your assistant](#connect-your-assistant) · [FAQ](#faq) ·
|
|
49
|
+
[Roadmap](#roadmap) · [Limitations](#limitations) · [Reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md)
|
|
50
|
+
|
|
51
|
+
## What you will see
|
|
52
|
+
|
|
53
|
+
**In your terminal or CI log**, `ridi-check gate` lists every answer that was right and is now wrong in
|
|
54
|
+
every run, and exits 1 so the pipeline stops:
|
|
55
|
+
|
|
56
|
+
```text
|
|
57
|
+
ridi-check gate: ridi-demo/ridi-out/results.csv, 30 cases, answer-key mode
|
|
58
|
+
stable_regression case q02
|
|
59
|
+
stable_regression case q09
|
|
60
|
+
stable_regression case q18
|
|
61
|
+
stable_regression case q25
|
|
62
|
+
counts: stable_regression=4 stable_improvement=1 regression_unconfirmed=0 improvement_unconfirmed=0 unstable=4 wrong_both=4 correct_both=17 incomplete=0
|
|
63
|
+
stable regressions: 4 (limit 0) - over the limit
|
|
64
|
+
gate: a limit was exceeded
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
**In the report**, each changed case shows the question, both answers, what the second run gave and the
|
|
68
|
+
passages only one version used:
|
|
69
|
+
|
|
70
|
+

|
|
71
|
+
|
|
72
|
+
Every question ends in one group:
|
|
73
|
+
|
|
74
|
+
| Group | Meaning |
|
|
75
|
+
|---|---|
|
|
76
|
+
| Stable regression | right before, wrong after, in every run: an answer your update broke |
|
|
77
|
+
| Stable improvement | wrong before, right after, in every run |
|
|
78
|
+
| Stable change | no answer key: a different answer in every run, for you to judge |
|
|
79
|
+
| Unstable on re-run | a version disagreed with itself, so the difference is not evidence about the update |
|
|
80
|
+
| Wrong in both | no change, still wrong; listed separately |
|
|
81
|
+
|
|
82
|
+
The report is one self-contained HTML file. It works offline, on a phone, in light and dark, in English and
|
|
83
|
+
Arabic. The numbers above come from `ridi-check demo`, which uses synthetic data.
|
|
84
|
+
|
|
85
|
+
## Who it is for
|
|
86
|
+
|
|
87
|
+
- **You are about to swap the model, edit the prompt or change retrieval** and want to know which answers
|
|
88
|
+
move, not only whether an average moved.
|
|
89
|
+
- **You have questions but no answer key.** The report shows what changed; you judge only the changes that
|
|
90
|
+
held on re-run.
|
|
91
|
+
- **You want a CI check** that fails when an answer that was right becomes wrong in every run.
|
|
92
|
+
- **You only have answers in a spreadsheet.** The [browser version](https://github.com/adeebnoor/ridi-check/blob/main/web/ridi-update-check.html) reads
|
|
93
|
+
pasted cells and needs no install.
|
|
94
|
+
|
|
95
|
+
It is **not** for proving that answers are correct or that a system is ready to deploy, for agents that use
|
|
96
|
+
tools or keep state between turns (not tested), or for explaining why an answer changed. See
|
|
97
|
+
[Limitations](#limitations) and the [use cases](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.md).
|
|
98
|
+
|
|
99
|
+
## Start in one minute
|
|
100
|
+
|
|
101
|
+
Python 3.10 or later, no dependencies. The demo needs no network and no API key; its data is synthetic.
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
pip install git+https://github.com/adeebnoor/ridi-check
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
ridi-check demo --open
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Then check your own assistant:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
ridi-check init my-check
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
It writes `questions.csv`, `my_assistant.py` (one function to fill in), `ridi-check.json` and
|
|
118
|
+
`ASK_YOUR_AI.md`, a prompt that lets your AI coding assistant do the wiring. In that folder, try five
|
|
119
|
+
questions with `ridi-check run --limit 5`, then run everything with `ridi-check run`. The
|
|
120
|
+
[user guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md) walks through each step and how to read the report.
|
|
121
|
+
|
|
122
|
+
**No install at all:** download [web/ridi-update-check.html](https://github.com/adeebnoor/ridi-check/blob/main/web/ridi-update-check.html), open it and
|
|
123
|
+
paste your results (CSV text or cells copied from a spreadsheet). The page makes no network request.
|
|
124
|
+
|
|
125
|
+
## Use it in CI
|
|
126
|
+
|
|
127
|
+
```yaml
|
|
128
|
+
- run: pip install git+https://github.com/adeebnoor/ridi-check
|
|
129
|
+
- run: ridi-check run --config ridi-check.json # calls both versions, writes ridi-out/results.csv
|
|
130
|
+
- run: ridi-check gate ridi-out/results.csv # exits 1 on a stable regression
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
A complete GitHub Actions workflow to copy: [examples/ci/ridi-check.yml](https://github.com/adeebnoor/ridi-check/blob/main/examples/ci/ridi-check.yml).
|
|
134
|
+
`gate` prints counts and the offending cases; it never prints a verdict about your system. Limits such as
|
|
135
|
+
`--max-stable-regressions 2` are in the [reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md#ridi-check-gate-resultscsv).
|
|
136
|
+
|
|
137
|
+
## Connect your assistant
|
|
138
|
+
|
|
139
|
+
ridi-check has no model provider built in. Your assistant is the system under test, reached through one
|
|
140
|
+
function:
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
def answer(question: str) -> dict:
|
|
144
|
+
return {"answer": "text", "context": ["doc-12", "doc-40"]} # context is optional
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
| Your assistant is… | Pass it as |
|
|
148
|
+
|---|---|
|
|
149
|
+
| importable from Python | `python:module:function` |
|
|
150
|
+
| a program in any language | `command:<shell command>` (question on stdin, JSON or text on stdout) |
|
|
151
|
+
| an HTTP service | `http://…`, or a JSON file that describes the request and the reply |
|
|
152
|
+
|
|
153
|
+
Ready-made examples in [examples/adapters](https://github.com/adeebnoor/ridi-check/blob/main/examples/adapters/README.md): an OpenAI-compatible HTTP
|
|
154
|
+
API (no code), the Claude API through its official SDK, and a shell command with `curl`. Each reads its key
|
|
155
|
+
from an environment variable and is tested against a local stand-in server.
|
|
156
|
+
|
|
157
|
+
The two versions should differ only in what the update changes. Details:
|
|
158
|
+
[developer guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.md#connecting-an-assistant).
|
|
159
|
+
|
|
160
|
+
## FAQ
|
|
161
|
+
|
|
162
|
+
**Do I need a test set or an answer key?** No. Without an `expected` column the report shows which answers
|
|
163
|
+
differ and you judge only the stable changes. With one, it speaks of regressions and improvements.
|
|
164
|
+
|
|
165
|
+
**Does my data go anywhere?** No. ridi-check calls only your assistant and writes local files. The report
|
|
166
|
+
and the browser version make no network request.
|
|
167
|
+
|
|
168
|
+
**Does it use a model to judge answers?** No. A judge adds its own error. Scoring is exact match after
|
|
169
|
+
normalisation, or a function you write.
|
|
170
|
+
|
|
171
|
+
**How many questions and runs?** 30 to 100 real user questions is a good first set. Two runs per version is
|
|
172
|
+
the minimum for the stable label; use more when the decision is expensive. Calls = questions × 2 × runs.
|
|
173
|
+
|
|
174
|
+
**Almost everything is "unstable" or "changed". Why?** Your assistant words the same answer differently each
|
|
175
|
+
time. Add `expected` answers with a scoring function, or use `--similarity 0.9`.
|
|
176
|
+
|
|
177
|
+
**Which providers and frameworks does it support?** Any. It talks to a function, a command or an HTTP
|
|
178
|
+
endpoint, not to a provider.
|
|
179
|
+
|
|
180
|
+
**Can I use it on an agent?** For its question-in, answer-out behaviour, yes. Agents that use tools or keep
|
|
181
|
+
state between turns have not been tested.
|
|
182
|
+
|
|
183
|
+
## Roadmap
|
|
184
|
+
|
|
185
|
+
No dates and no promises; this is what is planned and what is not.
|
|
186
|
+
|
|
187
|
+
- **Next:** a release on PyPI and a hosted link for the browser version.
|
|
188
|
+
- **Being considered:** intervals for the rates, multi-turn conversations, tool-using agents, more report
|
|
189
|
+
languages.
|
|
190
|
+
- **Not planned:** model-based judging, or any hosted service that receives your data.
|
|
191
|
+
|
|
192
|
+
Tell us what you need in an [issue](https://github.com/adeebnoor/ridi-check/issues).
|
|
193
|
+
|
|
194
|
+
## Limitations
|
|
195
|
+
|
|
196
|
+
- It finds change, not correctness. A version that is wrong the same way every time looks stable, and
|
|
197
|
+
cases wrong in both versions are listed separately.
|
|
198
|
+
- Without an answer key or your judgment, a difference is a change, not a regression.
|
|
199
|
+
- Passages are shown as they differ and are not graded; an ungraded passage can still carry the evidence.
|
|
200
|
+
- Two runs is a thin estimate of re-run variation. More runs make the stable label firmer.
|
|
201
|
+
- It is not a readiness or safety certificate. No output says a system is fit to deploy; the gate prints
|
|
202
|
+
counts and the decision stays yours.
|
|
203
|
+
- Not in this version: agents that use tools or keep state, model-based judging, statistical tests.
|
|
204
|
+
|
|
205
|
+
## Documentation
|
|
206
|
+
|
|
207
|
+
- [User guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md): step by step, and how to read the report
|
|
208
|
+
- [Use cases](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.md): twelve situations and how to set each one up
|
|
209
|
+
- [Developer guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.md): adapters, files, CI, library use
|
|
210
|
+
- [Reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md): every command and option, `results.csv`, the labelling rules
|
|
211
|
+
- بالعربية: [دليل الاستخدام](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.ar.md) · [حالات الاستخدام](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.ar.md) ·
|
|
212
|
+
[دليل المطورين](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.ar.md)
|
|
213
|
+
|
|
214
|
+
ridi-check follows the RIDI method.
|
|
215
|
+
|
|
216
|
+
## Contributing
|
|
217
|
+
|
|
218
|
+
Bug reports, questions and pull requests are welcome: see [CONTRIBUTING.md](https://github.com/adeebnoor/ridi-check/blob/main/CONTRIBUTING.md) and the
|
|
219
|
+
[changelog](https://github.com/adeebnoor/ridi-check/blob/main/CHANGELOG.md). Report a vulnerability privately, as [SECURITY.md](https://github.com/adeebnoor/ridi-check/blob/main/SECURITY.md)
|
|
220
|
+
describes.
|
|
221
|
+
|
|
222
|
+
## Licence
|
|
223
|
+
|
|
224
|
+
MIT. See [LICENSE](https://github.com/adeebnoor/ridi-check/blob/main/LICENSE).
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
# ridi-check
|
|
2
|
+
|
|
3
|
+
**See what a model or prompt update really changed in your assistant's answers, before you ship it.**
|
|
4
|
+
|
|
5
|
+
[](https://github.com/adeebnoor/ridi-check/actions/workflows/tests.yml)
|
|
6
|
+
[](https://github.com/adeebnoor/ridi-check/blob/main/LICENSE)
|
|
7
|
+
[](https://github.com/adeebnoor/ridi-check/blob/main/pyproject.toml)
|
|
8
|
+
|
|
9
|
+
A new model or an edited prompt can change what your assistant or agent answers without anyone noticing:
|
|
10
|
+
average accuracy barely moves while individual answers flip. ridi-check runs the old and the new version on
|
|
11
|
+
the same questions, more than once, and shows which answers changed and which of those changes hold on
|
|
12
|
+
re-run. You need no test set, no answer key and no model judge.
|
|
13
|
+
|
|
14
|
+

|
|
15
|
+
|
|
16
|
+
[What you will see](#what-you-will-see) · [Who it is for](#who-it-is-for) · [Start in one minute](#start-in-one-minute) ·
|
|
17
|
+
[Use it in CI](#use-it-in-ci) · [Connect your assistant](#connect-your-assistant) · [FAQ](#faq) ·
|
|
18
|
+
[Roadmap](#roadmap) · [Limitations](#limitations) · [Reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md)
|
|
19
|
+
|
|
20
|
+
## What you will see
|
|
21
|
+
|
|
22
|
+
**In your terminal or CI log**, `ridi-check gate` lists every answer that was right and is now wrong in
|
|
23
|
+
every run, and exits 1 so the pipeline stops:
|
|
24
|
+
|
|
25
|
+
```text
|
|
26
|
+
ridi-check gate: ridi-demo/ridi-out/results.csv, 30 cases, answer-key mode
|
|
27
|
+
stable_regression case q02
|
|
28
|
+
stable_regression case q09
|
|
29
|
+
stable_regression case q18
|
|
30
|
+
stable_regression case q25
|
|
31
|
+
counts: stable_regression=4 stable_improvement=1 regression_unconfirmed=0 improvement_unconfirmed=0 unstable=4 wrong_both=4 correct_both=17 incomplete=0
|
|
32
|
+
stable regressions: 4 (limit 0) - over the limit
|
|
33
|
+
gate: a limit was exceeded
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
**In the report**, each changed case shows the question, both answers, what the second run gave and the
|
|
37
|
+
passages only one version used:
|
|
38
|
+
|
|
39
|
+

|
|
40
|
+
|
|
41
|
+
Every question ends in one group:
|
|
42
|
+
|
|
43
|
+
| Group | Meaning |
|
|
44
|
+
|---|---|
|
|
45
|
+
| Stable regression | right before, wrong after, in every run: an answer your update broke |
|
|
46
|
+
| Stable improvement | wrong before, right after, in every run |
|
|
47
|
+
| Stable change | no answer key: a different answer in every run, for you to judge |
|
|
48
|
+
| Unstable on re-run | a version disagreed with itself, so the difference is not evidence about the update |
|
|
49
|
+
| Wrong in both | no change, still wrong; listed separately |
|
|
50
|
+
|
|
51
|
+
The report is one self-contained HTML file. It works offline, on a phone, in light and dark, in English and
|
|
52
|
+
Arabic. The numbers above come from `ridi-check demo`, which uses synthetic data.
|
|
53
|
+
|
|
54
|
+
## Who it is for
|
|
55
|
+
|
|
56
|
+
- **You are about to swap the model, edit the prompt or change retrieval** and want to know which answers
|
|
57
|
+
move, not only whether an average moved.
|
|
58
|
+
- **You have questions but no answer key.** The report shows what changed; you judge only the changes that
|
|
59
|
+
held on re-run.
|
|
60
|
+
- **You want a CI check** that fails when an answer that was right becomes wrong in every run.
|
|
61
|
+
- **You only have answers in a spreadsheet.** The [browser version](https://github.com/adeebnoor/ridi-check/blob/main/web/ridi-update-check.html) reads
|
|
62
|
+
pasted cells and needs no install.
|
|
63
|
+
|
|
64
|
+
It is **not** for proving that answers are correct or that a system is ready to deploy, for agents that use
|
|
65
|
+
tools or keep state between turns (not tested), or for explaining why an answer changed. See
|
|
66
|
+
[Limitations](#limitations) and the [use cases](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.md).
|
|
67
|
+
|
|
68
|
+
## Start in one minute
|
|
69
|
+
|
|
70
|
+
Python 3.10 or later, no dependencies. The demo needs no network and no API key; its data is synthetic.
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install git+https://github.com/adeebnoor/ridi-check
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
ridi-check demo --open
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Then check your own assistant:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
ridi-check init my-check
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
It writes `questions.csv`, `my_assistant.py` (one function to fill in), `ridi-check.json` and
|
|
87
|
+
`ASK_YOUR_AI.md`, a prompt that lets your AI coding assistant do the wiring. In that folder, try five
|
|
88
|
+
questions with `ridi-check run --limit 5`, then run everything with `ridi-check run`. The
|
|
89
|
+
[user guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md) walks through each step and how to read the report.
|
|
90
|
+
|
|
91
|
+
**No install at all:** download [web/ridi-update-check.html](https://github.com/adeebnoor/ridi-check/blob/main/web/ridi-update-check.html), open it and
|
|
92
|
+
paste your results (CSV text or cells copied from a spreadsheet). The page makes no network request.
|
|
93
|
+
|
|
94
|
+
## Use it in CI
|
|
95
|
+
|
|
96
|
+
```yaml
|
|
97
|
+
- run: pip install git+https://github.com/adeebnoor/ridi-check
|
|
98
|
+
- run: ridi-check run --config ridi-check.json # calls both versions, writes ridi-out/results.csv
|
|
99
|
+
- run: ridi-check gate ridi-out/results.csv # exits 1 on a stable regression
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
A complete GitHub Actions workflow to copy: [examples/ci/ridi-check.yml](https://github.com/adeebnoor/ridi-check/blob/main/examples/ci/ridi-check.yml).
|
|
103
|
+
`gate` prints counts and the offending cases; it never prints a verdict about your system. Limits such as
|
|
104
|
+
`--max-stable-regressions 2` are in the [reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md#ridi-check-gate-resultscsv).
|
|
105
|
+
|
|
106
|
+
## Connect your assistant
|
|
107
|
+
|
|
108
|
+
ridi-check has no model provider built in. Your assistant is the system under test, reached through one
|
|
109
|
+
function:
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
def answer(question: str) -> dict:
|
|
113
|
+
return {"answer": "text", "context": ["doc-12", "doc-40"]} # context is optional
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
| Your assistant is… | Pass it as |
|
|
117
|
+
|---|---|
|
|
118
|
+
| importable from Python | `python:module:function` |
|
|
119
|
+
| a program in any language | `command:<shell command>` (question on stdin, JSON or text on stdout) |
|
|
120
|
+
| an HTTP service | `http://…`, or a JSON file that describes the request and the reply |
|
|
121
|
+
|
|
122
|
+
Ready-made examples in [examples/adapters](https://github.com/adeebnoor/ridi-check/blob/main/examples/adapters/README.md): an OpenAI-compatible HTTP
|
|
123
|
+
API (no code), the Claude API through its official SDK, and a shell command with `curl`. Each reads its key
|
|
124
|
+
from an environment variable and is tested against a local stand-in server.
|
|
125
|
+
|
|
126
|
+
The two versions should differ only in what the update changes. Details:
|
|
127
|
+
[developer guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.md#connecting-an-assistant).
|
|
128
|
+
|
|
129
|
+
## FAQ
|
|
130
|
+
|
|
131
|
+
**Do I need a test set or an answer key?** No. Without an `expected` column the report shows which answers
|
|
132
|
+
differ and you judge only the stable changes. With one, it speaks of regressions and improvements.
|
|
133
|
+
|
|
134
|
+
**Does my data go anywhere?** No. ridi-check calls only your assistant and writes local files. The report
|
|
135
|
+
and the browser version make no network request.
|
|
136
|
+
|
|
137
|
+
**Does it use a model to judge answers?** No. A judge adds its own error. Scoring is exact match after
|
|
138
|
+
normalisation, or a function you write.
|
|
139
|
+
|
|
140
|
+
**How many questions and runs?** 30 to 100 real user questions is a good first set. Two runs per version is
|
|
141
|
+
the minimum for the stable label; use more when the decision is expensive. Calls = questions × 2 × runs.
|
|
142
|
+
|
|
143
|
+
**Almost everything is "unstable" or "changed". Why?** Your assistant words the same answer differently each
|
|
144
|
+
time. Add `expected` answers with a scoring function, or use `--similarity 0.9`.
|
|
145
|
+
|
|
146
|
+
**Which providers and frameworks does it support?** Any. It talks to a function, a command or an HTTP
|
|
147
|
+
endpoint, not to a provider.
|
|
148
|
+
|
|
149
|
+
**Can I use it on an agent?** For its question-in, answer-out behaviour, yes. Agents that use tools or keep
|
|
150
|
+
state between turns have not been tested.
|
|
151
|
+
|
|
152
|
+
## Roadmap
|
|
153
|
+
|
|
154
|
+
No dates and no promises; this is what is planned and what is not.
|
|
155
|
+
|
|
156
|
+
- **Next:** a release on PyPI and a hosted link for the browser version.
|
|
157
|
+
- **Being considered:** intervals for the rates, multi-turn conversations, tool-using agents, more report
|
|
158
|
+
languages.
|
|
159
|
+
- **Not planned:** model-based judging, or any hosted service that receives your data.
|
|
160
|
+
|
|
161
|
+
Tell us what you need in an [issue](https://github.com/adeebnoor/ridi-check/issues).
|
|
162
|
+
|
|
163
|
+
## Limitations
|
|
164
|
+
|
|
165
|
+
- It finds change, not correctness. A version that is wrong the same way every time looks stable, and
|
|
166
|
+
cases wrong in both versions are listed separately.
|
|
167
|
+
- Without an answer key or your judgment, a difference is a change, not a regression.
|
|
168
|
+
- Passages are shown as they differ and are not graded; an ungraded passage can still carry the evidence.
|
|
169
|
+
- Two runs is a thin estimate of re-run variation. More runs make the stable label firmer.
|
|
170
|
+
- It is not a readiness or safety certificate. No output says a system is fit to deploy; the gate prints
|
|
171
|
+
counts and the decision stays yours.
|
|
172
|
+
- Not in this version: agents that use tools or keep state, model-based judging, statistical tests.
|
|
173
|
+
|
|
174
|
+
## Documentation
|
|
175
|
+
|
|
176
|
+
- [User guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md): step by step, and how to read the report
|
|
177
|
+
- [Use cases](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.md): twelve situations and how to set each one up
|
|
178
|
+
- [Developer guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.md): adapters, files, CI, library use
|
|
179
|
+
- [Reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md): every command and option, `results.csv`, the labelling rules
|
|
180
|
+
- بالعربية: [دليل الاستخدام](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.ar.md) · [حالات الاستخدام](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.ar.md) ·
|
|
181
|
+
[دليل المطورين](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.ar.md)
|
|
182
|
+
|
|
183
|
+
ridi-check follows the RIDI method.
|
|
184
|
+
|
|
185
|
+
## Contributing
|
|
186
|
+
|
|
187
|
+
Bug reports, questions and pull requests are welcome: see [CONTRIBUTING.md](https://github.com/adeebnoor/ridi-check/blob/main/CONTRIBUTING.md) and the
|
|
188
|
+
[changelog](https://github.com/adeebnoor/ridi-check/blob/main/CHANGELOG.md). Report a vulnerability privately, as [SECURITY.md](https://github.com/adeebnoor/ridi-check/blob/main/SECURITY.md)
|
|
189
|
+
describes.
|
|
190
|
+
|
|
191
|
+
## Licence
|
|
192
|
+
|
|
193
|
+
MIT. See [LICENSE](https://github.com/adeebnoor/ridi-check/blob/main/LICENSE).
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ridi-check"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "See what a model or prompt update changed in an LLM assistant's answers, and which changes hold on re-run."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
dependencies = []
|
|
14
|
+
authors = [{ name = "ridi-check contributors" }]
|
|
15
|
+
keywords = ["llm", "evals", "regression-testing", "ai-agents", "prompt-testing", "rag", "ci"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Environment :: Console",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Programming Language :: Python :: 3.13",
|
|
27
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
28
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
29
|
+
"Topic :: Software Development :: Testing",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
test = ["pytest>=7"]
|
|
34
|
+
|
|
35
|
+
[project.urls]
|
|
36
|
+
Homepage = "https://github.com/adeebnoor/ridi-check"
|
|
37
|
+
Documentation = "https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md"
|
|
38
|
+
Changelog = "https://github.com/adeebnoor/ridi-check/blob/main/CHANGELOG.md"
|
|
39
|
+
Issues = "https://github.com/adeebnoor/ridi-check/issues"
|
|
40
|
+
|
|
41
|
+
[project.scripts]
|
|
42
|
+
ridi-check = "ridi_check.cli:main"
|
|
43
|
+
|
|
44
|
+
[tool.setuptools.dynamic]
|
|
45
|
+
version = { attr = "ridi_check.__version__" }
|
|
46
|
+
|
|
47
|
+
[tool.setuptools.packages.find]
|
|
48
|
+
where = ["src"]
|
|
49
|
+
|
|
50
|
+
[tool.setuptools.package-data]
|
|
51
|
+
ridi_check = ["report_template.html", "starter/*", "demo/*"]
|
|
52
|
+
|
|
53
|
+
[tool.pytest.ini_options]
|
|
54
|
+
testpaths = ["tests"]
|