realdelta 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- realdelta-0.1.0/.gitignore +25 -0
- realdelta-0.1.0/LICENSE +202 -0
- realdelta-0.1.0/PKG-INFO +343 -0
- realdelta-0.1.0/README.md +314 -0
- realdelta-0.1.0/docs/calibration_protocol.md +311 -0
- realdelta-0.1.0/examples/real_model/PROTOCOL.md +213 -0
- realdelta-0.1.0/examples/real_model/make_tasks.py +91 -0
- realdelta-0.1.0/examples/real_model/plan_scale.txt +22 -0
- realdelta-0.1.0/examples/real_model/run_20261116/a1.jsonl +200 -0
- realdelta-0.1.0/examples/real_model/run_20261116/a2.jsonl +200 -0
- realdelta-0.1.0/examples/real_model/run_20261116/b.jsonl +200 -0
- realdelta-0.1.0/examples/real_model/run_20261116/preflight.jsonl +2 -0
- realdelta-0.1.0/examples/real_model/run_20261116/provenance.txt +9 -0
- realdelta-0.1.0/examples/real_model/run_20261116/responses.jsonl +600 -0
- realdelta-0.1.0/examples/real_model/run_20261116/summary.txt +172 -0
- realdelta-0.1.0/examples/real_model/run_demo.py +399 -0
- realdelta-0.1.0/examples/real_model/tasks.jsonl +40 -0
- realdelta-0.1.0/examples/synthetic/baseline.jsonl +200 -0
- realdelta-0.1.0/examples/synthetic/regression.jsonl +200 -0
- realdelta-0.1.0/examples/synthetic/rerun.jsonl +200 -0
- realdelta-0.1.0/examples/synthetic/transcript.txt +50 -0
- realdelta-0.1.0/examples/synthetic_demo.py +139 -0
- realdelta-0.1.0/pyproject.toml +65 -0
- realdelta-0.1.0/results/calibration_grid_20261116.json +2275 -0
- realdelta-0.1.0/results/calibration_grid_20261116.log +127 -0
- realdelta-0.1.0/results/calibration_grid_20261116.md +166 -0
- realdelta-0.1.0/results/calibration_grid_20261116.provenance.txt +9 -0
- realdelta-0.1.0/results/calibration_grid_20261117.json +1468 -0
- realdelta-0.1.0/results/calibration_grid_20261117.log +74 -0
- realdelta-0.1.0/results/calibration_grid_20261117.md +110 -0
- realdelta-0.1.0/results/calibration_grid_20261117.provenance.txt +9 -0
- realdelta-0.1.0/results/delta_boundary_diagnostic_20261118.json +195 -0
- realdelta-0.1.0/results/delta_boundary_diagnostic_20261118.md +29 -0
- realdelta-0.1.0/results/suite_bounds_selection.json +192 -0
- realdelta-0.1.0/results/suite_bounds_selection.md +42 -0
- realdelta-0.1.0/results/suite_bounds_selection_a2.json +145 -0
- realdelta-0.1.0/results/suite_bounds_selection_a2.md +37 -0
- realdelta-0.1.0/scripts/calibration_grid.py +311 -0
- realdelta-0.1.0/scripts/delta_boundary_diagnostic.py +147 -0
- realdelta-0.1.0/scripts/suite_bounds_selection.py +287 -0
- realdelta-0.1.0/src/realdelta/__init__.py +37 -0
- realdelta-0.1.0/src/realdelta/__main__.py +5 -0
- realdelta-0.1.0/src/realdelta/cli.py +305 -0
- realdelta-0.1.0/src/realdelta/io.py +195 -0
- realdelta-0.1.0/src/realdelta/planner.py +137 -0
- realdelta-0.1.0/src/realdelta/simulate.py +51 -0
- realdelta-0.1.0/src/realdelta/stats.py +338 -0
- realdelta-0.1.0/tests/data/bad_json.jsonl +1 -0
- realdelta-0.1.0/tests/data/bad_passed.csv +2 -0
- realdelta-0.1.0/tests/data/bool_trial.jsonl +1 -0
- realdelta-0.1.0/tests/data/duplicate_key.jsonl +1 -0
- realdelta-0.1.0/tests/data/duplicate_trial.csv +3 -0
- realdelta-0.1.0/tests/data/empty.jsonl +0 -0
- realdelta-0.1.0/tests/data/excel_float_trial.csv +2 -0
- realdelta-0.1.0/tests/data/extra_column.csv +2 -0
- realdelta-0.1.0/tests/data/extra_key.jsonl +1 -0
- realdelta-0.1.0/tests/data/headerless.csv +2 -0
- realdelta-0.1.0/tests/data/int_passed.jsonl +1 -0
- realdelta-0.1.0/tests/data/latin1.jsonl +1 -0
- realdelta-0.1.0/tests/data/mixed_trial.jsonl +2 -0
- realdelta-0.1.0/tests/data/numeric_case.jsonl +1 -0
- realdelta-0.1.0/tests/data/ok.jsonl +4 -0
- realdelta-0.1.0/tests/data/ok.txt +2 -0
- realdelta-0.1.0/tests/data/ok_bom_crlf.csv +5 -0
- realdelta-0.1.0/tests/data/ok_reordered.csv +3 -0
- realdelta-0.1.0/tests/data/semicolon.csv +2 -0
- realdelta-0.1.0/tests/data/short_row.csv +2 -0
- realdelta-0.1.0/tests/data/string_passed.jsonl +1 -0
- realdelta-0.1.0/tests/data/utf16.csv +0 -0
- realdelta-0.1.0/tests/test_bounds_analytic.py +84 -0
- realdelta-0.1.0/tests/test_calibration.py +157 -0
- realdelta-0.1.0/tests/test_cli.py +273 -0
- realdelta-0.1.0/tests/test_demo.py +59 -0
- realdelta-0.1.0/tests/test_grid_truths.py +80 -0
- realdelta-0.1.0/tests/test_io.py +108 -0
- realdelta-0.1.0/tests/test_planner.py +92 -0
- realdelta-0.1.0/tests/test_planner_snapshot.py +21 -0
- realdelta-0.1.0/tests/test_readme_real_model.py +94 -0
- realdelta-0.1.0/tests/test_real_model_demo.py +423 -0
- realdelta-0.1.0/tests/test_stats.py +185 -0
- realdelta-0.1.0/tests/test_suite_analytic.py +120 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Secrets: API keys live in environment variables only
|
|
2
|
+
.env
|
|
3
|
+
.env.*
|
|
4
|
+
*.key
|
|
5
|
+
|
|
6
|
+
# Python
|
|
7
|
+
__pycache__/
|
|
8
|
+
*.py[cod]
|
|
9
|
+
.venv/
|
|
10
|
+
venv/
|
|
11
|
+
build/
|
|
12
|
+
dist/
|
|
13
|
+
*.egg-info/
|
|
14
|
+
.pytest_cache/
|
|
15
|
+
.coverage
|
|
16
|
+
htmlcov/
|
|
17
|
+
|
|
18
|
+
# Editors and OS
|
|
19
|
+
.vscode/
|
|
20
|
+
.idea/
|
|
21
|
+
.DS_Store
|
|
22
|
+
Thumbs.db
|
|
23
|
+
|
|
24
|
+
# Grid smoke runs (not evidence)
|
|
25
|
+
smoke_results/
|
realdelta-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
|
202
|
+
|
realdelta-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: realdelta
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Is that change in your agent eval real, or noise? Intervals, paired comparisons and trial planning for repeated agent evals; interval coverage checked on a pre-registered simulation grid.
|
|
5
|
+
Project-URL: Repository, https://github.com/althea-yuquan-chen/realdelta
|
|
6
|
+
Project-URL: Issues, https://github.com/althea-yuquan-chen/realdelta/issues
|
|
7
|
+
Author: Althea Yuquan Chen
|
|
8
|
+
License-Expression: Apache-2.0
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: a/b-testing,agents,confidence-intervals,evaluation,llm,statistics
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Mathematics
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: numpy>=1.24
|
|
23
|
+
Requires-Dist: scipy>=1.10
|
|
24
|
+
Provides-Extra: demo
|
|
25
|
+
Requires-Dist: openai>=3.37; extra == 'demo'
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# realdelta
|
|
31
|
+
|
|
32
|
+
Is that change in your agent eval real, or noise?
|
|
33
|
+
|
|
34
|
+
You changed a prompt, a tool description or the model, re-ran your eval suite, and
|
|
35
|
+
the pass rate moved from 82% to 86%. `realdelta` tells you whether that move is
|
|
36
|
+
larger than the run-to-run noise of your agent, and how many repeated trials you
|
|
37
|
+
need before it can tell.
|
|
38
|
+
|
|
39
|
+
**Status:** version 0.1.0, alpha. The statistics
|
|
40
|
+
core, its calibration checks, result-file loading, the command line, a synthetic demo
|
|
41
|
+
and one pre-registered demo on a real model are done. The test suite runs in CI on
|
|
42
|
+
Linux and Windows, Python 3.10 to 3.13. Links below point to the files of this release.
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
Python 3.10 or later; depends on numpy and scipy.
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install realdelta
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The tests, demos and calibration scripts live in the repository, so to run them install from a clone:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
git clone https://github.com/althea-yuquan-chen/realdelta
|
|
56
|
+
cd realdelta
|
|
57
|
+
pip install -e ".[dev]" # the dev extra adds pytest
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## What it does
|
|
61
|
+
|
|
62
|
+
- **Pass-rate intervals for repeated trials, in two scopes.** `suite` asks how sure
|
|
63
|
+
we are about *this fixed set of cases*: given each case's pass rate, its trials
|
|
64
|
+
are independent draws, so only trial-to-trial noise counts. `population` asks how
|
|
65
|
+
sure we are about *cases like these*: the case is the unit, and case-to-case
|
|
66
|
+
spread counts too.
|
|
67
|
+
- **Paired comparisons** (suite scope). Baseline and candidate run on the same
|
|
68
|
+
cases; the change gets an interval and a sign-flip permutation p-value.
|
|
69
|
+
`result.detected` means the interval excludes 0; it does not use the p-value, and
|
|
70
|
+
the two can disagree.
|
|
71
|
+
- **Trial planning.** Simulates a synthetic suite you parameterise (case pass rates
|
|
72
|
+
from a Beta distribution) under one of two changes: every case shifted by the same
|
|
73
|
+
amount, or k randomly chosen cases broken (pass rate 0), the common agent
|
|
74
|
+
regression. It reports power and false-alarm rate, with Monte Carlo standard
|
|
75
|
+
errors, for 1, 3, 5, 10 or 20 trials per case, and the realised change: shifts are
|
|
76
|
+
clipped at 0 and 1, and a broken case removes its own pass rate. A separate
|
|
77
|
+
function estimates cost from prices you supply.
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from realdelta import compare, pass_rate_interval
|
|
81
|
+
|
|
82
|
+
baseline = {"case-1": [True, True, False, True, True], "case-2": [...], ...}
|
|
83
|
+
candidate = {"case-1": [True, True, True, True, True], "case-2": [...], ...}
|
|
84
|
+
|
|
85
|
+
print(pass_rate_interval(candidate, scope="suite"))
|
|
86
|
+
result = compare(baseline, candidate) # suite scope
|
|
87
|
+
print(result.delta, result.detected, result.p_value)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## Result files
|
|
91
|
+
|
|
92
|
+
One record per trial, as JSON Lines (`.jsonl`) or CSV (`.csv`), UTF-8:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
{"case_id": "case-001", "passed": true}
|
|
96
|
+
{"case_id": "case-001", "passed": false}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
case_id,passed
|
|
101
|
+
case-001,true
|
|
102
|
+
case-001,false
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Fields: `case_id` (a non-empty string), `passed` (a boolean; in CSV one of true,
|
|
106
|
+
false, 1, 0) and, optionally, `trial` (a whole number, on every line or on none;
|
|
107
|
+
each case and trial pair must be unique). Any other field is rejected. File-format
|
|
108
|
+
error messages name the file, line and field, never the value; when `compare` finds
|
|
109
|
+
cases present in only one file, it shows up to three of their ids, cut to 40
|
|
110
|
+
characters. Record failed, crashed or
|
|
111
|
+
timed-out runs as failures: dropping or re-running a trial because of its outcome
|
|
112
|
+
breaks assumption A4 below. Without `trial`, a file concatenated twice cannot be
|
|
113
|
+
detected, so every command prints the number of cases and trials it read; check it.
|
|
114
|
+
`load_outcomes(path)` reads a file into the dictionary the Python API takes.
|
|
115
|
+
|
|
116
|
+
## Command line
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
python -m realdelta interval results.jsonl # pass rate with its interval
|
|
120
|
+
python -m realdelta compare baseline.jsonl candidate.jsonl # paired change, suite scope
|
|
121
|
+
python -m realdelta plan --cases 40 --delta -0.05 # power per trials option
|
|
122
|
+
python -m realdelta plan --cases 40 --break-cases 2 --trials 3,5,10 \
|
|
123
|
+
--input-tokens 3000 --output-tokens 500 --usd-per-m-input 1 --usd-per-m-output 5
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
After installing, `realdelta ...` is the same command as `python -m realdelta ...`.
|
|
127
|
+
Add `--json` for one JSON object. Reports from `compare` and `plan` (and
|
|
128
|
+
from `interval` in the population scope) end with notes on what the result cannot
|
|
129
|
+
tell you; warnings (on stderr) flag settings outside the tested range below, such
|
|
130
|
+
as fewer than 10 cases or fewer than 3 trials per case for a comparison. `compare`
|
|
131
|
+
uses seed 0 for its permutation p-value (with more than 16 cases; up to 16 it
|
|
132
|
+
enumerates every sign flip) unless you pass `--seed`.
|
|
133
|
+
|
|
134
|
+
## Synthetic demo
|
|
135
|
+
|
|
136
|
+
[`examples/synthetic_demo.py`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/examples/synthetic_demo.py) simulates an agent with 40
|
|
137
|
+
cases that mostly pass (case pass rates from Beta(20,1); true pass rate 0.95507), run 5
|
|
138
|
+
times each, with one fixed seed (the first one tried) and no model calls. Its full
|
|
139
|
+
output is in [`examples/synthetic/transcript.txt`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/examples/synthetic/transcript.txt);
|
|
140
|
+
a test re-runs it and compares the output byte for byte. The comparison runs only with
|
|
141
|
+
numpy 2.4.4, the version the transcript was recorded with (other versions skip it).
|
|
142
|
+
The CI workflow pins 2.4.4, and the comparison has passed there on Linux and Windows
|
|
143
|
+
with Python 3.11, 3.12 and 3.13.
|
|
144
|
+
|
|
145
|
+
- **Noise alone moves the number.** Across 500 pairs of runs of the *unchanged*
|
|
146
|
+
agent, the pass rate moved by 2 points or more in 184 pairs (36.8%) and by 4 points
|
|
147
|
+
or more in 35 (7.0%); the 95th percentile of the move is 4.0 points. `compare`
|
|
148
|
+
flagged a change in 23 of the 500 (4.6%); its sign-flip p-value was below 0.05 in
|
|
149
|
+
14 (2.8%).
|
|
150
|
+
- **A regression in a few cases is caught by the interval much more often than by
|
|
151
|
+
the p-value.**
|
|
152
|
+
In a candidate where 3 of the 40 cases fail every time (true change -0.073), the
|
|
153
|
+
interval excluded 0 in 423 of 500 pairs of runs (84.6%); the sign-flip p-value was
|
|
154
|
+
below 0.05 in 76 (15.2%). The sign-flip test asks whether cases moved in a
|
|
155
|
+
consistent direction across the suite, and only 3 cases moved; the interval
|
|
156
|
+
measures the trial-to-trial noise of this fixed suite, which is the suite scope's
|
|
157
|
+
question. A permutation test within cases would match that scope; it is not
|
|
158
|
+
implemented.
|
|
159
|
+
- **How many trials?** `plan` for suites like this one (40 cases, Beta(20,1), 3 cases
|
|
160
|
+
broken; it draws a new suite in every simulation) gives a simulated power of
|
|
161
|
+
0.13 +- 0.02 with 1 trial per case (below the gated range), 0.54 +- 0.03 with 3,
|
|
162
|
+
0.85 +- 0.02 with 5 and 0.99 +- 0.01 with 10.
|
|
163
|
+
|
|
164
|
+
These are simulation results for this setting, not calibration claims.
|
|
165
|
+
|
|
166
|
+
## Demo on a real model
|
|
167
|
+
|
|
168
|
+
[`examples/real_model/`](https://github.com/althea-yuquan-chen/realdelta/tree/v0.1.0/examples/real_model/) runs 40 generated puzzles (arithmetic, letter
|
|
169
|
+
counting, weekdays, sorting; 10 each) on GPT-5.6 Luna served through Azure OpenAI, with reasoning
|
|
170
|
+
effort low. Azure's content filtering is part of the system measured. Everything that decides the
|
|
171
|
+
result was fixed in [`PROTOCOL.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/examples/real_model/PROTOCOL.md) before any paid call; its
|
|
172
|
+
Amendment 1 changed the model from Claude Haiku 5.5 to GPT-5.6 Luna before any call was made. Three
|
|
173
|
+
arms, 5 trials per case each, 600 calls in one interleaved order: A1 and A2 use the same system prompt
|
|
174
|
+
(an A/A pair), B adds "Answer immediately; do not show your working." The run
|
|
175
|
+
([`run_20261116/summary.txt`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/examples/real_model/run_20261116/summary.txt)) used 51,213 input and
|
|
176
|
+
52,408 output tokens with no re-issued calls, timeouts, refusals or truncated outputs. The run
|
|
177
|
+
directory is named after the local date; it started at 2026-11-17T00:54Z.
|
|
178
|
+
|
|
179
|
+
**What "pass" measures here.** A trial passes only if its value is right *and* its last line has the
|
|
180
|
+
form `ANSWER: <value>`, as the pre-registered grader requires. The prompts carry two format
|
|
181
|
+
instructions side by side: the system prompt asks for that line, the user message says "Give the
|
|
182
|
+
answer in this form: ...". Most failures in this run are a correct value without the prefix, so the
|
|
183
|
+
pass rates below mostly measure compliance with the answer format, not puzzle-solving accuracy.
|
|
184
|
+
|
|
185
|
+
| Arm | Pass rate (suite scope) | 95% interval |
|
|
186
|
+
|---|---|---|
|
|
187
|
+
| A1 | 0.875 | [0.821, 0.917] |
|
|
188
|
+
| A2 | 0.855 | [0.798, 0.901] |
|
|
189
|
+
| B | 0.820 | [0.760, 0.871] |
|
|
190
|
+
|
|
191
|
+
- **A1 vs A2:** -0.020, interval [-0.086, +0.046]. The A/A pair was not flagged. It is one
|
|
192
|
+
realisation: in the synthetic demo, 4.6% of unchanged pairs were flagged.
|
|
193
|
+
- **A1 vs B:** -0.055. No change detected at this scale (40 cases, 5 trials); the interval
|
|
194
|
+
[-0.123, +0.014] bounds the change. Given the point above, what this comparison measures is
|
|
195
|
+
mostly format compliance, not accuracy.
|
|
196
|
+
- **Failures per arm** (pass / wrong value / no `ANSWER:` line; no other category occurred):
|
|
197
|
+
A1 175 / 1 / 24, A2 171 / 1 / 28, B 164 / 2 / 34. The grader checks the answer line first, so a
|
|
198
|
+
wrong value without the prefix counts as "no `ANSWER:` line".
|
|
199
|
+
- **Population scope:** because the cases are random draws from the puzzle generator (10 per family;
|
|
200
|
+
treating that stratified sample as independent draws is usually conservative), A1's pass rate for
|
|
201
|
+
puzzles from this generator is 0.875 with interval [0.732, 0.958].
|
|
202
|
+
|
|
203
|
+
Descriptive, not pre-registered, and not tested:
|
|
204
|
+
|
|
205
|
+
- **Value correctness ignoring the prefix:** reading the last line's value with or without `ANSWER:`,
|
|
206
|
+
A1 has 196 of 200 trials right, A2 199 and B 197. 82 of the 86 "no `ANSWER:` line" trials carry the
|
|
207
|
+
right value.
|
|
208
|
+
- **Failures by puzzle family** (any category):
|
|
209
|
+
|
|
210
|
+
| Family | A1 | A2 | B |
|
|
211
|
+
|---|---|---|---|
|
|
212
|
+
| arithmetic | 9 | 6 | 3 |
|
|
213
|
+
| letters | 0 | 5 | 21 |
|
|
214
|
+
| weekday | 15 | 17 | 6 |
|
|
215
|
+
| sort | 1 | 1 | 6 |
|
|
216
|
+
|
|
217
|
+
In this run B failed more on letter counting and sorting and less on weekdays and arithmetic, so
|
|
218
|
+
per-family differences partly cancel in the overall rate. No per-family difference was tested.
|
|
219
|
+
- **Flaky cases:** 15 to 20 cases per arm passed on some trials and failed on others.
|
|
220
|
+
|
|
221
|
+
A1 appears in both comparisons, so they are not independent, and each can raise a false alarm (4.6%
|
|
222
|
+
of unchanged pairs in the synthetic demo; the calibration tests allow up to 7%). The task set is
|
|
223
|
+
synthetic and says nothing about other tasks. Dollar cost is not quoted here because Azure's price
|
|
224
|
+
for this model is uncertain; `summary.txt` gives an upper estimate.
|
|
225
|
+
|
|
226
|
+
## How the intervals were checked
|
|
227
|
+
|
|
228
|
+
The calibration grid (its settings, the 0.93 gate and the rules for failures) was
|
|
229
|
+
pre-registered in [`docs/calibration_protocol.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/docs/calibration_protocol.md)
|
|
230
|
+
before any final result existed. The suite pass-rate method was replaced after the
|
|
231
|
+
first final run failed; the steps are below. Each grid row simulates suites with
|
|
232
|
+
known case pass rates and counts how often the 95% interval contains the truth.
|
|
233
|
+
|
|
234
|
+
**Claim.** On the grid run [`results/calibration_grid_20261117.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/calibration_grid_20261117.md)
|
|
235
|
+
(2000 repetitions per row, so a Monte Carlo standard error of about 0.005; 95% level),
|
|
236
|
+
every gated row has an estimated coverage of at least 0.93. The lowest rows are
|
|
237
|
+
0.956 for the suite pass rate (36 rows), 0.951 for the population pass rate (40 rows)
|
|
238
|
+
and 0.945 for the suite delta (25 rows). This is a statement about the tested
|
|
239
|
+
settings, not a guarantee of 95% coverage elsewhere.
|
|
240
|
+
|
|
241
|
+
**Tested settings** (gated rows; only the 95% level):
|
|
242
|
+
|
|
243
|
+
| Quantity | Cases | Trials per case | Case pass rates |
|
|
244
|
+
|---|---|---|---|
|
|
245
|
+
| Suite pass rate | 5 to 50 | 1 to 10, and unequal {1, 3, 10} | Beta(0.5,0.5), (2,2), (4,2), (20,1), (50,1); two mixtures; homogeneous 0.5 and 0.9 |
|
|
246
|
+
| Population pass rate | 5 to 50 | 1 to 10, and unequal {1, 3, 10} | as above, plus a 95%/5% pass/fail mixture and sticky trials (see Limitations) |
|
|
247
|
+
| Suite delta | 10 to 50 | 3 to 10, and unequal {1, 3, 10} at 30 cases | Beta(4,2) and Beta(20,1); no change, +0.05 and +0.10 |
|
|
248
|
+
|
|
249
|
+
Swapping baseline and candidate negates the delta interval exactly (a test pins
|
|
250
|
+
this), so the +0.05 and +0.10 delta rows also cover drops of the same realised size
|
|
251
|
+
from the shifted baseline; they do not cover drops from Beta(4,2) or Beta(20,1)
|
|
252
|
+
themselves, nor a few cases breaking outright.
|
|
253
|
+
The table gives ranges, not a full cross of every combination;
|
|
254
|
+
[`results/calibration_grid_20261117.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/calibration_grid_20261117.md) lists every row.
|
|
255
|
+
File names carry the seed base (20261116, 20261117, 20261118), not a date; both grid
|
|
256
|
+
runs started on 2026-11-16 (see their provenance files).
|
|
257
|
+
|
|
258
|
+
**How we got there.** This history is part of the evidence, so it is listed in full.
|
|
259
|
+
|
|
260
|
+
1. The first final grid ([`results/calibration_grid_20261116.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/calibration_grid_20261116.md))
|
|
261
|
+
failed one suite row and the population-delta width rule. With 5 cases, 1 trial
|
|
262
|
+
each and rates near 100%, the Wilson interval then used covered 0.917: a single
|
|
263
|
+
failure pulls its upper bound below the truth.
|
|
264
|
+
2. Amendment 1 pre-registered an exact analytic check (minimum coverage >= 0.947)
|
|
265
|
+
and two candidates: Clopper-Pearson on the effective sample size, and Wilson
|
|
266
|
+
with a continuity correction. Both failed it, at 0.9449 and 0.9434
|
|
267
|
+
([`results/suite_bounds_selection.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/suite_bounds_selection.md)).
|
|
268
|
+
3. Amendment 2 added one candidate: Clopper-Pearson with the effective sample size
|
|
269
|
+
capped at the number of trials run. It was designed *after* seeing step 2's
|
|
270
|
+
results, so the analytic check is not independent evidence for it. It passed
|
|
271
|
+
that check with 0.9503 ([`results/suite_bounds_selection_a2.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/suite_bounds_selection_a2.md)).
|
|
272
|
+
The independent evidence is the CI coverage tests, whose gates did not change,
|
|
273
|
+
and the 20261117 grid, which ran once after the implementation.
|
|
274
|
+
|
|
275
|
+
**The cost: wider, often conservative intervals.** Suite pass-rate intervals are
|
|
276
|
+
about 1.11 times as wide as the Wilson intervals of the 20261116 run (median across
|
|
277
|
+
rows, range 1.05 to 1.24; the two runs used different seeds, so the ratio is noisy).
|
|
278
|
+
Seven of the 36 suite pass-rate rows cover more than 99.5% (up to 0.999). With equal
|
|
279
|
+
trials per case the suite interval is Clopper-Pearson on the pooled pass count, which
|
|
280
|
+
is conservative when most cases are deterministic. The population pass rate is more
|
|
281
|
+
conservative still: if all m cases pass, its lower bound is 0.025^(1/m) (0.83 for 20
|
|
282
|
+
cases) however many trials each case ran, because m cases cannot rule out a failure
|
|
283
|
+
mode affecting a few percent of cases like them.
|
|
284
|
+
|
|
285
|
+
## Limitations
|
|
286
|
+
|
|
287
|
+
- **No population-scope delta in v0.1.** It failed the pre-registered width rule in
|
|
288
|
+
the 20261116 run (mean width 0.368 and 0.364 against a limit of 0.30), so
|
|
289
|
+
`delta_interval(scope="population")`, `compare(scope="population")` and
|
|
290
|
+
`plan_trials(scope="population")` raise `NotImplementedError`.
|
|
291
|
+
- **The suite delta is gated only for 10 to 50 cases with 3 to 10 trials each**
|
|
292
|
+
(plus one unequal-trials setting). It combines the two arms' Wilson-based
|
|
293
|
+
intervals (MOVER), and Wilson bounds under-cover near 100%. A reported-only
|
|
294
|
+
diagnostic ([`results/delta_boundary_diagnostic_20261118.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/delta_boundary_diagnostic_20261118.md))
|
|
295
|
+
covers part of the gap near 100%: with case rates from Beta(50,1) or Beta(20,1),
|
|
296
|
+
5 or 20 cases and 1 or 3 trials each, under no change, a 0.05 drop and (Beta(20,1)
|
|
297
|
+
only) a 0.05 rise, coverage was at least 0.971 across 20 settings. Three cases
|
|
298
|
+
built by hand to be adverse (an all-pass baseline against a candidate just below
|
|
299
|
+
the Wilson bound; not verified to be the worst) reached at least 0.949. Settings
|
|
300
|
+
outside these, such as moderate pass rates with 1 trial per case or fewer than 10
|
|
301
|
+
cases, have no evidence for the delta. `plan_trials` offers 1 trial per case by
|
|
302
|
+
default, which falls in that untested region. Power near 100% is low: a 5-point
|
|
303
|
+
drop is detected in at most 16.8% of runs (20 cases, 3 trials each), and in at most
|
|
304
|
+
5.1% with 1 trial per case.
|
|
305
|
+
- **The pass-rate and delta intervals use different bounds** (Clopper-Pearson and
|
|
306
|
+
Wilson), so one cannot be derived from the other on the same data.
|
|
307
|
+
- **Unequal trial counts** are covered by three grid settings (two pass-rate, one
|
|
308
|
+
delta) and one exact, reported-only suite pass-rate configuration (5 cases with
|
|
309
|
+
1, 1, 3, 3 and 10 trials, homogeneous rate: minimum coverage 0.958).
|
|
310
|
+
- **Assumptions.** A1: trials within a case are independent and identically
|
|
311
|
+
distributed. A2: cases are an independent draw from a population. A3: the two arms
|
|
312
|
+
share no randomness (seeds, caches, outage windows). A4: the number of trials per
|
|
313
|
+
case does not depend on outcomes. The suite scope needs A1, A3 and A4; the
|
|
314
|
+
population pass rate needs A2 and A4. The grid includes violations, reported and
|
|
315
|
+
not gated: trials that repeat the previous outcome (violates A1) drop suite
|
|
316
|
+
pass-rate coverage to 0.871 and 0.734; a shared per-trial shock in each arm
|
|
317
|
+
(violates A3) drops suite delta coverage to 0.894, with a false-alarm rate of
|
|
318
|
+
0.105. A2 holds only if your cases were sampled from the population you care
|
|
319
|
+
about; for a hand-picked suite, the population scope describes a hypothetical
|
|
320
|
+
population of similar cases.
|
|
321
|
+
- **Not checked:** confidence levels other than 95%, and the calibration of the
|
|
322
|
+
p-value from `compare`. It tests the sharp null "the two versions behave
|
|
323
|
+
identically on every case", and in the synthetic demo's setting (3 of 40 cases
|
|
324
|
+
broken) it rejected in 15.2% of runs against 84.6% for the interval.
|
|
325
|
+
|
|
326
|
+
## Reproduce
|
|
327
|
+
|
|
328
|
+
After installing from a clone (see Install), from the repository root:
|
|
329
|
+
|
|
330
|
+
```bash
|
|
331
|
+
python -m pytest # unit, analytic and CI coverage tests
|
|
332
|
+
python scripts/calibration_grid.py --smoke # 50 repetitions; not evidence
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
Both final grid runs are committed in `results/` with a provenance file, and each
|
|
336
|
+
ran once; the diagnostic records its commit in its own header. The grid script
|
|
337
|
+
refuses to overwrite existing results, and in this release it uses seed base
|
|
338
|
+
20261117; the 20261116 run is reproducible from commit da182ba. The selection and diagnostic scripts are `scripts/suite_bounds_selection.py`
|
|
339
|
+
and `scripts/delta_boundary_diagnostic.py`.
|
|
340
|
+
|
|
341
|
+
## License
|
|
342
|
+
|
|
343
|
+
Apache-2.0
|