ridi-check 0.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. ridi_check-0.3.2/LICENSE +21 -0
  2. ridi_check-0.3.2/MANIFEST.in +2 -0
  3. ridi_check-0.3.2/PKG-INFO +224 -0
  4. ridi_check-0.3.2/README.md +193 -0
  5. ridi_check-0.3.2/pyproject.toml +54 -0
  6. ridi_check-0.3.2/setup.cfg +4 -0
  7. ridi_check-0.3.2/src/ridi_check/__init__.py +3 -0
  8. ridi_check-0.3.2/src/ridi_check/__main__.py +6 -0
  9. ridi_check-0.3.2/src/ridi_check/adapters.py +257 -0
  10. ridi_check-0.3.2/src/ridi_check/analysis.py +190 -0
  11. ridi_check-0.3.2/src/ridi_check/cli.py +337 -0
  12. ridi_check-0.3.2/src/ridi_check/contract.py +222 -0
  13. ridi_check-0.3.2/src/ridi_check/demo/questions.csv +31 -0
  14. ridi_check-0.3.2/src/ridi_check/demo/ridi-check.json +11 -0
  15. ridi_check-0.3.2/src/ridi_check/demo/toy_assistant.py +154 -0
  16. ridi_check-0.3.2/src/ridi_check/gate.py +76 -0
  17. ridi_check-0.3.2/src/ridi_check/report.py +412 -0
  18. ridi_check-0.3.2/src/ridi_check/report_template.html +321 -0
  19. ridi_check-0.3.2/src/ridi_check/runner.py +501 -0
  20. ridi_check-0.3.2/src/ridi_check/scoring.py +21 -0
  21. ridi_check-0.3.2/src/ridi_check/starter/ASK_YOUR_AI.md +36 -0
  22. ridi_check-0.3.2/src/ridi_check/starter/my_assistant.py +37 -0
  23. ridi_check-0.3.2/src/ridi_check/starter/questions.csv +4 -0
  24. ridi_check-0.3.2/src/ridi_check/starter/results_template.csv +4 -0
  25. ridi_check-0.3.2/src/ridi_check/starter/ridi-check.json +10 -0
  26. ridi_check-0.3.2/src/ridi_check/strings.py +242 -0
  27. ridi_check-0.3.2/src/ridi_check/util.py +70 -0
  28. ridi_check-0.3.2/src/ridi_check.egg-info/PKG-INFO +224 -0
  29. ridi_check-0.3.2/src/ridi_check.egg-info/SOURCES.txt +31 -0
  30. ridi_check-0.3.2/src/ridi_check.egg-info/dependency_links.txt +1 -0
  31. ridi_check-0.3.2/src/ridi_check.egg-info/entry_points.txt +2 -0
  32. ridi_check-0.3.2/src/ridi_check.egg-info/requires.txt +3 -0
  33. ridi_check-0.3.2/src/ridi_check.egg-info/top_level.txt +1 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 ridi-check contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,2 @@
1
+ # The tests need files that are not part of the package (examples, docs, web); they live in the repository.
2
+ prune tests
@@ -0,0 +1,224 @@
1
+ Metadata-Version: 2.4
2
+ Name: ridi-check
3
+ Version: 0.3.2
4
+ Summary: See what a model or prompt update changed in an LLM assistant's answers, and which changes hold on re-run.
5
+ Author: ridi-check contributors
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/adeebnoor/ridi-check
8
+ Project-URL: Documentation, https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md
9
+ Project-URL: Changelog, https://github.com/adeebnoor/ridi-check/blob/main/CHANGELOG.md
10
+ Project-URL: Issues, https://github.com/adeebnoor/ridi-check/issues
11
+ Keywords: llm,evals,regression-testing,ai-agents,prompt-testing,rag,ci
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
23
+ Classifier: Topic :: Software Development :: Quality Assurance
24
+ Classifier: Topic :: Software Development :: Testing
25
+ Requires-Python: >=3.10
26
+ Description-Content-Type: text/markdown
27
+ License-File: LICENSE
28
+ Provides-Extra: test
29
+ Requires-Dist: pytest>=7; extra == "test"
30
+ Dynamic: license-file
31
+
32
+ # ridi-check
33
+
34
+ **See what a model or prompt update really changed in your assistant's answers, before you ship it.**
35
+
36
+ [![tests](https://github.com/adeebnoor/ridi-check/actions/workflows/tests.yml/badge.svg)](https://github.com/adeebnoor/ridi-check/actions/workflows/tests.yml)
37
+ [![licence: MIT](https://raw.githubusercontent.com/adeebnoor/ridi-check/main/docs/img/badge-licence.svg)](https://github.com/adeebnoor/ridi-check/blob/main/LICENSE)
38
+ [![Python 3.10+](https://raw.githubusercontent.com/adeebnoor/ridi-check/main/docs/img/badge-python.svg)](https://github.com/adeebnoor/ridi-check/blob/main/pyproject.toml)
39
+
40
+ A new model or an edited prompt can change what your assistant or agent answers without anyone noticing:
41
+ average accuracy barely moves while individual answers flip. ridi-check runs the old and the new version on
42
+ the same questions, more than once, and shows which answers changed and which of those changes hold on
43
+ re-run. You need no test set, no answer key and no model judge.
44
+
45
+ ![The report: where each case landed, and the change between versions against re-run variation](https://raw.githubusercontent.com/adeebnoor/ridi-check/main/docs/img/report.png)
46
+
47
+ [What you will see](#what-you-will-see) · [Who it is for](#who-it-is-for) · [Start in one minute](#start-in-one-minute) ·
48
+ [Use it in CI](#use-it-in-ci) · [Connect your assistant](#connect-your-assistant) · [FAQ](#faq) ·
49
+ [Roadmap](#roadmap) · [Limitations](#limitations) · [Reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md)
50
+
51
+ ## What you will see
52
+
53
+ **In your terminal or CI log**, `ridi-check gate` lists every answer that was right and is now wrong in
54
+ every run, and exits 1 so the pipeline stops:
55
+
56
+ ```text
57
+ ridi-check gate: ridi-demo/ridi-out/results.csv, 30 cases, answer-key mode
58
+ stable_regression case q02
59
+ stable_regression case q09
60
+ stable_regression case q18
61
+ stable_regression case q25
62
+ counts: stable_regression=4 stable_improvement=1 regression_unconfirmed=0 improvement_unconfirmed=0 unstable=4 wrong_both=4 correct_both=17 incomplete=0
63
+ stable regressions: 4 (limit 0) - over the limit
64
+ gate: a limit was exceeded
65
+ ```
66
+
67
+ **In the report**, each changed case shows the question, both answers, what the second run gave and the
68
+ passages only one version used:
69
+
70
+ ![One case to review: a stable regression, both answers, and the passages that differ](https://raw.githubusercontent.com/adeebnoor/ridi-check/main/docs/img/report-case.png)
71
+
72
+ Every question ends in one group:
73
+
74
+ | Group | Meaning |
75
+ |---|---|
76
+ | Stable regression | right before, wrong after, in every run: an answer your update broke |
77
+ | Stable improvement | wrong before, right after, in every run |
78
+ | Stable change | no answer key: a different answer in every run, for you to judge |
79
+ | Unstable on re-run | a version disagreed with itself, so the difference is not evidence about the update |
80
+ | Wrong in both | no change, still wrong; listed separately |
81
+
82
+ The report is one self-contained HTML file. It works offline, on a phone, in light and dark, in English and
83
+ Arabic. The numbers above come from `ridi-check demo`, which uses synthetic data.
84
+
85
+ ## Who it is for
86
+
87
+ - **You are about to swap the model, edit the prompt or change retrieval** and want to know which answers
88
+ move, not only whether an average moved.
89
+ - **You have questions but no answer key.** The report shows what changed; you judge only the changes that
90
+ held on re-run.
91
+ - **You want a CI check** that fails when an answer that was right becomes wrong in every run.
92
+ - **You only have answers in a spreadsheet.** The [browser version](https://github.com/adeebnoor/ridi-check/blob/main/web/ridi-update-check.html) reads
93
+ pasted cells and needs no install.
94
+
95
+ It is **not** for proving that answers are correct or that a system is ready to deploy, for agents that use
96
+ tools or keep state between turns (not tested), or for explaining why an answer changed. See
97
+ [Limitations](#limitations) and the [use cases](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.md).
98
+
99
+ ## Start in one minute
100
+
101
+ Python 3.10 or later, no dependencies. The demo needs no network and no API key; its data is synthetic.
102
+
103
+ ```bash
104
+ pip install git+https://github.com/adeebnoor/ridi-check
105
+ ```
106
+
107
+ ```bash
108
+ ridi-check demo --open
109
+ ```
110
+
111
+ Then check your own assistant:
112
+
113
+ ```bash
114
+ ridi-check init my-check
115
+ ```
116
+
117
+ It writes `questions.csv`, `my_assistant.py` (one function to fill in), `ridi-check.json` and
118
+ `ASK_YOUR_AI.md`, a prompt that lets your AI coding assistant do the wiring. In that folder, try five
119
+ questions with `ridi-check run --limit 5`, then run everything with `ridi-check run`. The
120
+ [user guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md) walks through each step and how to read the report.
121
+
122
+ **No install at all:** download [web/ridi-update-check.html](https://github.com/adeebnoor/ridi-check/blob/main/web/ridi-update-check.html), open it and
123
+ paste your results (CSV text or cells copied from a spreadsheet). The page makes no network request.
124
+
125
+ ## Use it in CI
126
+
127
+ ```yaml
128
+ - run: pip install git+https://github.com/adeebnoor/ridi-check
129
+ - run: ridi-check run --config ridi-check.json # calls both versions, writes ridi-out/results.csv
130
+ - run: ridi-check gate ridi-out/results.csv # exits 1 on a stable regression
131
+ ```
132
+
133
+ A complete GitHub Actions workflow to copy: [examples/ci/ridi-check.yml](https://github.com/adeebnoor/ridi-check/blob/main/examples/ci/ridi-check.yml).
134
+ `gate` prints counts and the offending cases; it never prints a verdict about your system. Limits such as
135
+ `--max-stable-regressions 2` are in the [reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md#ridi-check-gate-resultscsv).
136
+
137
+ ## Connect your assistant
138
+
139
+ ridi-check has no model provider built in. Your assistant is the system under test, reached through one
140
+ function:
141
+
142
+ ```python
143
+ def answer(question: str) -> dict:
144
+ return {"answer": "text", "context": ["doc-12", "doc-40"]} # context is optional
145
+ ```
146
+
147
+ | Your assistant is… | Pass it as |
148
+ |---|---|
149
+ | importable from Python | `python:module:function` |
150
+ | a program in any language | `command:<shell command>` (question on stdin, JSON or text on stdout) |
151
+ | an HTTP service | `http://…`, or a JSON file that describes the request and the reply |
152
+
153
+ Ready-made examples in [examples/adapters](https://github.com/adeebnoor/ridi-check/blob/main/examples/adapters/README.md): an OpenAI-compatible HTTP
154
+ API (no code), the Claude API through its official SDK, and a shell command with `curl`. Each reads its key
155
+ from an environment variable and is tested against a local stand-in server.
156
+
157
+ The two versions should differ only in what the update changes. Details:
158
+ [developer guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.md#connecting-an-assistant).
159
+
160
+ ## FAQ
161
+
162
+ **Do I need a test set or an answer key?** No. Without an `expected` column the report shows which answers
163
+ differ and you judge only the stable changes. With one, it speaks of regressions and improvements.
164
+
165
+ **Does my data go anywhere?** No. ridi-check calls only your assistant and writes local files. The report
166
+ and the browser version make no network request.
167
+
168
+ **Does it use a model to judge answers?** No. A judge adds its own error. Scoring is exact match after
169
+ normalisation, or a function you write.
170
+
171
+ **How many questions and runs?** 30 to 100 real user questions is a good first set. Two runs per version is
172
+ the minimum for the stable label; use more when the decision is expensive. Calls = questions × 2 × runs.
173
+
174
+ **Almost everything is "unstable" or "changed". Why?** Your assistant words the same answer differently each
175
+ time. Add `expected` answers with a scoring function, or use `--similarity 0.9`.
176
+
177
+ **Which providers and frameworks does it support?** Any. It talks to a function, a command or an HTTP
178
+ endpoint, not to a provider.
179
+
180
+ **Can I use it on an agent?** For its question-in, answer-out behaviour, yes. Agents that use tools or keep
181
+ state between turns have not been tested.
182
+
183
+ ## Roadmap
184
+
185
+ No dates and no promises; this is what is planned and what is not.
186
+
187
+ - **Next:** a release on PyPI and a hosted link for the browser version.
188
+ - **Being considered:** intervals for the rates, multi-turn conversations, tool-using agents, more report
189
+ languages.
190
+ - **Not planned:** model-based judging, or any hosted service that receives your data.
191
+
192
+ Tell us what you need in an [issue](https://github.com/adeebnoor/ridi-check/issues).
193
+
194
+ ## Limitations
195
+
196
+ - It finds change, not correctness. A version that is wrong the same way every time looks stable, and
197
+ cases wrong in both versions are listed separately.
198
+ - Without an answer key or your judgment, a difference is a change, not a regression.
199
+ - Passages are shown as they differ and are not graded; an ungraded passage can still carry the evidence.
200
+ - Two runs is a thin estimate of re-run variation. More runs make the stable label firmer.
201
+ - It is not a readiness or safety certificate. No output says a system is fit to deploy; the gate prints
202
+ counts and the decision stays yours.
203
+ - Not in this version: agents that use tools or keep state, model-based judging, statistical tests.
204
+
205
+ ## Documentation
206
+
207
+ - [User guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md): step by step, and how to read the report
208
+ - [Use cases](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.md): twelve situations and how to set each one up
209
+ - [Developer guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.md): adapters, files, CI, library use
210
+ - [Reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md): every command and option, `results.csv`, the labelling rules
211
+ - بالعربية: [دليل الاستخدام](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.ar.md) · [حالات الاستخدام](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.ar.md) ·
212
+ [دليل المطورين](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.ar.md)
213
+
214
+ ridi-check follows the RIDI method.
215
+
216
+ ## Contributing
217
+
218
+ Bug reports, questions and pull requests are welcome: see [CONTRIBUTING.md](https://github.com/adeebnoor/ridi-check/blob/main/CONTRIBUTING.md) and the
219
+ [changelog](https://github.com/adeebnoor/ridi-check/blob/main/CHANGELOG.md). Report a vulnerability privately, as [SECURITY.md](https://github.com/adeebnoor/ridi-check/blob/main/SECURITY.md)
220
+ describes.
221
+
222
+ ## Licence
223
+
224
+ MIT. See [LICENSE](https://github.com/adeebnoor/ridi-check/blob/main/LICENSE).
@@ -0,0 +1,193 @@
1
+ # ridi-check
2
+
3
+ **See what a model or prompt update really changed in your assistant's answers, before you ship it.**
4
+
5
+ [![tests](https://github.com/adeebnoor/ridi-check/actions/workflows/tests.yml/badge.svg)](https://github.com/adeebnoor/ridi-check/actions/workflows/tests.yml)
6
+ [![licence: MIT](https://raw.githubusercontent.com/adeebnoor/ridi-check/main/docs/img/badge-licence.svg)](https://github.com/adeebnoor/ridi-check/blob/main/LICENSE)
7
+ [![Python 3.10+](https://raw.githubusercontent.com/adeebnoor/ridi-check/main/docs/img/badge-python.svg)](https://github.com/adeebnoor/ridi-check/blob/main/pyproject.toml)
8
+
9
+ A new model or an edited prompt can change what your assistant or agent answers without anyone noticing:
10
+ average accuracy barely moves while individual answers flip. ridi-check runs the old and the new version on
11
+ the same questions, more than once, and shows which answers changed and which of those changes hold on
12
+ re-run. You need no test set, no answer key and no model judge.
13
+
14
+ ![The report: where each case landed, and the change between versions against re-run variation](https://raw.githubusercontent.com/adeebnoor/ridi-check/main/docs/img/report.png)
15
+
16
+ [What you will see](#what-you-will-see) · [Who it is for](#who-it-is-for) · [Start in one minute](#start-in-one-minute) ·
17
+ [Use it in CI](#use-it-in-ci) · [Connect your assistant](#connect-your-assistant) · [FAQ](#faq) ·
18
+ [Roadmap](#roadmap) · [Limitations](#limitations) · [Reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md)
19
+
20
+ ## What you will see
21
+
22
+ **In your terminal or CI log**, `ridi-check gate` lists every answer that was right and is now wrong in
23
+ every run, and exits 1 so the pipeline stops:
24
+
25
+ ```text
26
+ ridi-check gate: ridi-demo/ridi-out/results.csv, 30 cases, answer-key mode
27
+ stable_regression case q02
28
+ stable_regression case q09
29
+ stable_regression case q18
30
+ stable_regression case q25
31
+ counts: stable_regression=4 stable_improvement=1 regression_unconfirmed=0 improvement_unconfirmed=0 unstable=4 wrong_both=4 correct_both=17 incomplete=0
32
+ stable regressions: 4 (limit 0) - over the limit
33
+ gate: a limit was exceeded
34
+ ```
35
+
36
+ **In the report**, each changed case shows the question, both answers, what the second run gave and the
37
+ passages only one version used:
38
+
39
+ ![One case to review: a stable regression, both answers, and the passages that differ](https://raw.githubusercontent.com/adeebnoor/ridi-check/main/docs/img/report-case.png)
40
+
41
+ Every question ends in one group:
42
+
43
+ | Group | Meaning |
44
+ |---|---|
45
+ | Stable regression | right before, wrong after, in every run: an answer your update broke |
46
+ | Stable improvement | wrong before, right after, in every run |
47
+ | Stable change | no answer key: a different answer in every run, for you to judge |
48
+ | Unstable on re-run | a version disagreed with itself, so the difference is not evidence about the update |
49
+ | Wrong in both | no change, still wrong; listed separately |
50
+
51
+ The report is one self-contained HTML file. It works offline, on a phone, in light and dark, in English and
52
+ Arabic. The numbers above come from `ridi-check demo`, which uses synthetic data.
53
+
54
+ ## Who it is for
55
+
56
+ - **You are about to swap the model, edit the prompt or change retrieval** and want to know which answers
57
+ move, not only whether an average moved.
58
+ - **You have questions but no answer key.** The report shows what changed; you judge only the changes that
59
+ held on re-run.
60
+ - **You want a CI check** that fails when an answer that was right becomes wrong in every run.
61
+ - **You only have answers in a spreadsheet.** The [browser version](https://github.com/adeebnoor/ridi-check/blob/main/web/ridi-update-check.html) reads
62
+ pasted cells and needs no install.
63
+
64
+ It is **not** for proving that answers are correct or that a system is ready to deploy, for agents that use
65
+ tools or keep state between turns (not tested), or for explaining why an answer changed. See
66
+ [Limitations](#limitations) and the [use cases](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.md).
67
+
68
+ ## Start in one minute
69
+
70
+ Python 3.10 or later, no dependencies. The demo needs no network and no API key; its data is synthetic.
71
+
72
+ ```bash
73
+ pip install git+https://github.com/adeebnoor/ridi-check
74
+ ```
75
+
76
+ ```bash
77
+ ridi-check demo --open
78
+ ```
79
+
80
+ Then check your own assistant:
81
+
82
+ ```bash
83
+ ridi-check init my-check
84
+ ```
85
+
86
+ It writes `questions.csv`, `my_assistant.py` (one function to fill in), `ridi-check.json` and
87
+ `ASK_YOUR_AI.md`, a prompt that lets your AI coding assistant do the wiring. In that folder, try five
88
+ questions with `ridi-check run --limit 5`, then run everything with `ridi-check run`. The
89
+ [user guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md) walks through each step and how to read the report.
90
+
91
+ **No install at all:** download [web/ridi-update-check.html](https://github.com/adeebnoor/ridi-check/blob/main/web/ridi-update-check.html), open it and
92
+ paste your results (CSV text or cells copied from a spreadsheet). The page makes no network request.
93
+
94
+ ## Use it in CI
95
+
96
+ ```yaml
97
+ - run: pip install git+https://github.com/adeebnoor/ridi-check
98
+ - run: ridi-check run --config ridi-check.json # calls both versions, writes ridi-out/results.csv
99
+ - run: ridi-check gate ridi-out/results.csv # exits 1 on a stable regression
100
+ ```
101
+
102
+ A complete GitHub Actions workflow to copy: [examples/ci/ridi-check.yml](https://github.com/adeebnoor/ridi-check/blob/main/examples/ci/ridi-check.yml).
103
+ `gate` prints counts and the offending cases; it never prints a verdict about your system. Limits such as
104
+ `--max-stable-regressions 2` are in the [reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md#ridi-check-gate-resultscsv).
105
+
106
+ ## Connect your assistant
107
+
108
+ ridi-check has no model provider built in. Your assistant is the system under test, reached through one
109
+ function:
110
+
111
+ ```python
112
+ def answer(question: str) -> dict:
113
+ return {"answer": "text", "context": ["doc-12", "doc-40"]} # context is optional
114
+ ```
115
+
116
+ | Your assistant is… | Pass it as |
117
+ |---|---|
118
+ | importable from Python | `python:module:function` |
119
+ | a program in any language | `command:<shell command>` (question on stdin, JSON or text on stdout) |
120
+ | an HTTP service | `http://…`, or a JSON file that describes the request and the reply |
121
+
122
+ Ready-made examples in [examples/adapters](https://github.com/adeebnoor/ridi-check/blob/main/examples/adapters/README.md): an OpenAI-compatible HTTP
123
+ API (no code), the Claude API through its official SDK, and a shell command with `curl`. Each reads its key
124
+ from an environment variable and is tested against a local stand-in server.
125
+
126
+ The two versions should differ only in what the update changes. Details:
127
+ [developer guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.md#connecting-an-assistant).
128
+
129
+ ## FAQ
130
+
131
+ **Do I need a test set or an answer key?** No. Without an `expected` column the report shows which answers
132
+ differ and you judge only the stable changes. With one, it speaks of regressions and improvements.
133
+
134
+ **Does my data go anywhere?** No. ridi-check calls only your assistant and writes local files. The report
135
+ and the browser version make no network request.
136
+
137
+ **Does it use a model to judge answers?** No. A judge adds its own error. Scoring is exact match after
138
+ normalisation, or a function you write.
139
+
140
+ **How many questions and runs?** 30 to 100 real user questions is a good first set. Two runs per version is
141
+ the minimum for the stable label; use more when the decision is expensive. Calls = questions × 2 × runs.
142
+
143
+ **Almost everything is "unstable" or "changed". Why?** Your assistant words the same answer differently each
144
+ time. Add `expected` answers with a scoring function, or use `--similarity 0.9`.
145
+
146
+ **Which providers and frameworks does it support?** Any. It talks to a function, a command or an HTTP
147
+ endpoint, not to a provider.
148
+
149
+ **Can I use it on an agent?** For its question-in, answer-out behaviour, yes. Agents that use tools or keep
150
+ state between turns have not been tested.
151
+
152
+ ## Roadmap
153
+
154
+ No dates and no promises; this is what is planned and what is not.
155
+
156
+ - **Next:** a release on PyPI and a hosted link for the browser version.
157
+ - **Being considered:** intervals for the rates, multi-turn conversations, tool-using agents, more report
158
+ languages.
159
+ - **Not planned:** model-based judging, or any hosted service that receives your data.
160
+
161
+ Tell us what you need in an [issue](https://github.com/adeebnoor/ridi-check/issues).
162
+
163
+ ## Limitations
164
+
165
+ - It finds change, not correctness. A version that is wrong the same way every time looks stable, and
166
+ cases wrong in both versions are listed separately.
167
+ - Without an answer key or your judgment, a difference is a change, not a regression.
168
+ - Passages are shown as they differ and are not graded; an ungraded passage can still carry the evidence.
169
+ - Two runs is a thin estimate of re-run variation. More runs make the stable label firmer.
170
+ - It is not a readiness or safety certificate. No output says a system is fit to deploy; the gate prints
171
+ counts and the decision stays yours.
172
+ - Not in this version: agents that use tools or keep state, model-based judging, statistical tests.
173
+
174
+ ## Documentation
175
+
176
+ - [User guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md): step by step, and how to read the report
177
+ - [Use cases](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.md): twelve situations and how to set each one up
178
+ - [Developer guide](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.md): adapters, files, CI, library use
179
+ - [Reference](https://github.com/adeebnoor/ridi-check/blob/main/docs/REFERENCE.md): every command and option, `results.csv`, the labelling rules
180
+ - بالعربية: [دليل الاستخدام](https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.ar.md) · [حالات الاستخدام](https://github.com/adeebnoor/ridi-check/blob/main/docs/USE_CASES.ar.md) ·
181
+ [دليل المطورين](https://github.com/adeebnoor/ridi-check/blob/main/docs/DEVELOPER_GUIDE.ar.md)
182
+
183
+ ridi-check follows the RIDI method.
184
+
185
+ ## Contributing
186
+
187
+ Bug reports, questions and pull requests are welcome: see [CONTRIBUTING.md](https://github.com/adeebnoor/ridi-check/blob/main/CONTRIBUTING.md) and the
188
+ [changelog](https://github.com/adeebnoor/ridi-check/blob/main/CHANGELOG.md). Report a vulnerability privately, as [SECURITY.md](https://github.com/adeebnoor/ridi-check/blob/main/SECURITY.md)
189
+ describes.
190
+
191
+ ## Licence
192
+
193
+ MIT. See [LICENSE](https://github.com/adeebnoor/ridi-check/blob/main/LICENSE).
@@ -0,0 +1,54 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "ridi-check"
7
+ dynamic = ["version"]
8
+ description = "See what a model or prompt update changed in an LLM assistant's answers, and which changes hold on re-run."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ dependencies = []
14
+ authors = [{ name = "ridi-check contributors" }]
15
+ keywords = ["llm", "evals", "regression-testing", "ai-agents", "prompt-testing", "rag", "ci"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Environment :: Console",
19
+ "Intended Audience :: Developers",
20
+ "Operating System :: OS Independent",
21
+ "Programming Language :: Python :: 3",
22
+ "Programming Language :: Python :: 3 :: Only",
23
+ "Programming Language :: Python :: 3.10",
24
+ "Programming Language :: Python :: 3.11",
25
+ "Programming Language :: Python :: 3.12",
26
+ "Programming Language :: Python :: 3.13",
27
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
28
+ "Topic :: Software Development :: Quality Assurance",
29
+ "Topic :: Software Development :: Testing",
30
+ ]
31
+
32
+ [project.optional-dependencies]
33
+ test = ["pytest>=7"]
34
+
35
+ [project.urls]
36
+ Homepage = "https://github.com/adeebnoor/ridi-check"
37
+ Documentation = "https://github.com/adeebnoor/ridi-check/blob/main/docs/USER_GUIDE.md"
38
+ Changelog = "https://github.com/adeebnoor/ridi-check/blob/main/CHANGELOG.md"
39
+ Issues = "https://github.com/adeebnoor/ridi-check/issues"
40
+
41
+ [project.scripts]
42
+ ridi-check = "ridi_check.cli:main"
43
+
44
+ [tool.setuptools.dynamic]
45
+ version = { attr = "ridi_check.__version__" }
46
+
47
+ [tool.setuptools.packages.find]
48
+ where = ["src"]
49
+
50
+ [tool.setuptools.package-data]
51
+ ridi_check = ["report_template.html", "starter/*", "demo/*"]
52
+
53
+ [tool.pytest.ini_options]
54
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,3 @@
1
+ """ridi-check: what did an update change, and which changes hold on re-run."""
2
+
3
+ __version__ = "0.3.2"
@@ -0,0 +1,6 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ if __name__ == "__main__":
6
+ sys.exit(main())