realdelta 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. realdelta-0.1.0/.gitignore +25 -0
  2. realdelta-0.1.0/LICENSE +202 -0
  3. realdelta-0.1.0/PKG-INFO +343 -0
  4. realdelta-0.1.0/README.md +314 -0
  5. realdelta-0.1.0/docs/calibration_protocol.md +311 -0
  6. realdelta-0.1.0/examples/real_model/PROTOCOL.md +213 -0
  7. realdelta-0.1.0/examples/real_model/make_tasks.py +91 -0
  8. realdelta-0.1.0/examples/real_model/plan_scale.txt +22 -0
  9. realdelta-0.1.0/examples/real_model/run_20261116/a1.jsonl +200 -0
  10. realdelta-0.1.0/examples/real_model/run_20261116/a2.jsonl +200 -0
  11. realdelta-0.1.0/examples/real_model/run_20261116/b.jsonl +200 -0
  12. realdelta-0.1.0/examples/real_model/run_20261116/preflight.jsonl +2 -0
  13. realdelta-0.1.0/examples/real_model/run_20261116/provenance.txt +9 -0
  14. realdelta-0.1.0/examples/real_model/run_20261116/responses.jsonl +600 -0
  15. realdelta-0.1.0/examples/real_model/run_20261116/summary.txt +172 -0
  16. realdelta-0.1.0/examples/real_model/run_demo.py +399 -0
  17. realdelta-0.1.0/examples/real_model/tasks.jsonl +40 -0
  18. realdelta-0.1.0/examples/synthetic/baseline.jsonl +200 -0
  19. realdelta-0.1.0/examples/synthetic/regression.jsonl +200 -0
  20. realdelta-0.1.0/examples/synthetic/rerun.jsonl +200 -0
  21. realdelta-0.1.0/examples/synthetic/transcript.txt +50 -0
  22. realdelta-0.1.0/examples/synthetic_demo.py +139 -0
  23. realdelta-0.1.0/pyproject.toml +65 -0
  24. realdelta-0.1.0/results/calibration_grid_20261116.json +2275 -0
  25. realdelta-0.1.0/results/calibration_grid_20261116.log +127 -0
  26. realdelta-0.1.0/results/calibration_grid_20261116.md +166 -0
  27. realdelta-0.1.0/results/calibration_grid_20261116.provenance.txt +9 -0
  28. realdelta-0.1.0/results/calibration_grid_20261117.json +1468 -0
  29. realdelta-0.1.0/results/calibration_grid_20261117.log +74 -0
  30. realdelta-0.1.0/results/calibration_grid_20261117.md +110 -0
  31. realdelta-0.1.0/results/calibration_grid_20261117.provenance.txt +9 -0
  32. realdelta-0.1.0/results/delta_boundary_diagnostic_20261118.json +195 -0
  33. realdelta-0.1.0/results/delta_boundary_diagnostic_20261118.md +29 -0
  34. realdelta-0.1.0/results/suite_bounds_selection.json +192 -0
  35. realdelta-0.1.0/results/suite_bounds_selection.md +42 -0
  36. realdelta-0.1.0/results/suite_bounds_selection_a2.json +145 -0
  37. realdelta-0.1.0/results/suite_bounds_selection_a2.md +37 -0
  38. realdelta-0.1.0/scripts/calibration_grid.py +311 -0
  39. realdelta-0.1.0/scripts/delta_boundary_diagnostic.py +147 -0
  40. realdelta-0.1.0/scripts/suite_bounds_selection.py +287 -0
  41. realdelta-0.1.0/src/realdelta/__init__.py +37 -0
  42. realdelta-0.1.0/src/realdelta/__main__.py +5 -0
  43. realdelta-0.1.0/src/realdelta/cli.py +305 -0
  44. realdelta-0.1.0/src/realdelta/io.py +195 -0
  45. realdelta-0.1.0/src/realdelta/planner.py +137 -0
  46. realdelta-0.1.0/src/realdelta/simulate.py +51 -0
  47. realdelta-0.1.0/src/realdelta/stats.py +338 -0
  48. realdelta-0.1.0/tests/data/bad_json.jsonl +1 -0
  49. realdelta-0.1.0/tests/data/bad_passed.csv +2 -0
  50. realdelta-0.1.0/tests/data/bool_trial.jsonl +1 -0
  51. realdelta-0.1.0/tests/data/duplicate_key.jsonl +1 -0
  52. realdelta-0.1.0/tests/data/duplicate_trial.csv +3 -0
  53. realdelta-0.1.0/tests/data/empty.jsonl +0 -0
  54. realdelta-0.1.0/tests/data/excel_float_trial.csv +2 -0
  55. realdelta-0.1.0/tests/data/extra_column.csv +2 -0
  56. realdelta-0.1.0/tests/data/extra_key.jsonl +1 -0
  57. realdelta-0.1.0/tests/data/headerless.csv +2 -0
  58. realdelta-0.1.0/tests/data/int_passed.jsonl +1 -0
  59. realdelta-0.1.0/tests/data/latin1.jsonl +1 -0
  60. realdelta-0.1.0/tests/data/mixed_trial.jsonl +2 -0
  61. realdelta-0.1.0/tests/data/numeric_case.jsonl +1 -0
  62. realdelta-0.1.0/tests/data/ok.jsonl +4 -0
  63. realdelta-0.1.0/tests/data/ok.txt +2 -0
  64. realdelta-0.1.0/tests/data/ok_bom_crlf.csv +5 -0
  65. realdelta-0.1.0/tests/data/ok_reordered.csv +3 -0
  66. realdelta-0.1.0/tests/data/semicolon.csv +2 -0
  67. realdelta-0.1.0/tests/data/short_row.csv +2 -0
  68. realdelta-0.1.0/tests/data/string_passed.jsonl +1 -0
  69. realdelta-0.1.0/tests/data/utf16.csv +0 -0
  70. realdelta-0.1.0/tests/test_bounds_analytic.py +84 -0
  71. realdelta-0.1.0/tests/test_calibration.py +157 -0
  72. realdelta-0.1.0/tests/test_cli.py +273 -0
  73. realdelta-0.1.0/tests/test_demo.py +59 -0
  74. realdelta-0.1.0/tests/test_grid_truths.py +80 -0
  75. realdelta-0.1.0/tests/test_io.py +108 -0
  76. realdelta-0.1.0/tests/test_planner.py +92 -0
  77. realdelta-0.1.0/tests/test_planner_snapshot.py +21 -0
  78. realdelta-0.1.0/tests/test_readme_real_model.py +94 -0
  79. realdelta-0.1.0/tests/test_real_model_demo.py +423 -0
  80. realdelta-0.1.0/tests/test_stats.py +185 -0
  81. realdelta-0.1.0/tests/test_suite_analytic.py +120 -0
@@ -0,0 +1,25 @@
1
+ # Secrets: API keys live in environment variables only
2
+ .env
3
+ .env.*
4
+ *.key
5
+
6
+ # Python
7
+ __pycache__/
8
+ *.py[cod]
9
+ .venv/
10
+ venv/
11
+ build/
12
+ dist/
13
+ *.egg-info/
14
+ .pytest_cache/
15
+ .coverage
16
+ htmlcov/
17
+
18
+ # Editors and OS
19
+ .vscode/
20
+ .idea/
21
+ .DS_Store
22
+ Thumbs.db
23
+
24
+ # Grid smoke runs (not evidence)
25
+ smoke_results/
@@ -0,0 +1,202 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
202
+
@@ -0,0 +1,343 @@
1
+ Metadata-Version: 2.5
2
+ Name: realdelta
3
+ Version: 0.1.0
4
+ Summary: Is that change in your agent eval real, or noise? Intervals, paired comparisons and trial planning for repeated agent evals; interval coverage checked on a pre-registered simulation grid.
5
+ Project-URL: Repository, https://github.com/althea-yuquan-chen/realdelta
6
+ Project-URL: Issues, https://github.com/althea-yuquan-chen/realdelta/issues
7
+ Author: Althea Yuquan Chen
8
+ License-Expression: Apache-2.0
9
+ License-File: LICENSE
10
+ Keywords: a/b-testing,agents,confidence-intervals,evaluation,llm,statistics
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Scientific/Engineering :: Mathematics
20
+ Classifier: Topic :: Software Development :: Testing
21
+ Requires-Python: >=3.10
22
+ Requires-Dist: numpy>=1.24
23
+ Requires-Dist: scipy>=1.10
24
+ Provides-Extra: demo
25
+ Requires-Dist: openai>=3.37; extra == 'demo'
26
+ Provides-Extra: dev
27
+ Requires-Dist: pytest>=8; extra == 'dev'
28
+ Description-Content-Type: text/markdown
29
+
30
+ # realdelta
31
+
32
+ Is that change in your agent eval real, or noise?
33
+
34
+ You changed a prompt, a tool description or the model, re-ran your eval suite, and
35
+ the pass rate moved from 82% to 86%. `realdelta` tells you whether that move is
36
+ larger than the run-to-run noise of your agent, and how many repeated trials you
37
+ need before it can tell.
38
+
39
+ **Status:** version 0.1.0, alpha. The statistics
40
+ core, its calibration checks, result-file loading, the command line, a synthetic demo
41
+ and one pre-registered demo on a real model are done. The test suite runs in CI on
42
+ Linux and Windows, Python 3.10 to 3.13. Links below point to the files of this release.
43
+
44
+ ## Install
45
+
46
+ Python 3.10 or later; depends on numpy and scipy.
47
+
48
+ ```bash
49
+ pip install realdelta
50
+ ```
51
+
52
+ The tests, demos and calibration scripts live in the repository, so to run them install from a clone:
53
+
54
+ ```bash
55
+ git clone https://github.com/althea-yuquan-chen/realdelta
56
+ cd realdelta
57
+ pip install -e ".[dev]" # the dev extra adds pytest
58
+ ```
59
+
60
+ ## What it does
61
+
62
+ - **Pass-rate intervals for repeated trials, in two scopes.** `suite` asks how sure
63
+ we are about *this fixed set of cases*: given each case's pass rate, its trials
64
+ are independent draws, so only trial-to-trial noise counts. `population` asks how
65
+ sure we are about *cases like these*: the case is the unit, and case-to-case
66
+ spread counts too.
67
+ - **Paired comparisons** (suite scope). Baseline and candidate run on the same
68
+ cases; the change gets an interval and a sign-flip permutation p-value.
69
+ `result.detected` means the interval excludes 0; it does not use the p-value, and
70
+ the two can disagree.
71
+ - **Trial planning.** Simulates a synthetic suite you parameterise (case pass rates
72
+ from a Beta distribution) under one of two changes: every case shifted by the same
73
+ amount, or k randomly chosen cases broken (pass rate 0), the common agent
74
+ regression. It reports power and false-alarm rate, with Monte Carlo standard
75
+ errors, for 1, 3, 5, 10 or 20 trials per case, and the realised change: shifts are
76
+ clipped at 0 and 1, and a broken case removes its own pass rate. A separate
77
+ function estimates cost from prices you supply.
78
+
79
+ ```python
80
+ from realdelta import compare, pass_rate_interval
81
+
82
+ baseline = {"case-1": [True, True, False, True, True], "case-2": [...], ...}
83
+ candidate = {"case-1": [True, True, True, True, True], "case-2": [...], ...}
84
+
85
+ print(pass_rate_interval(candidate, scope="suite"))
86
+ result = compare(baseline, candidate) # suite scope
87
+ print(result.delta, result.detected, result.p_value)
88
+ ```
89
+
90
+ ## Result files
91
+
92
+ One record per trial, as JSON Lines (`.jsonl`) or CSV (`.csv`), UTF-8:
93
+
94
+ ```
95
+ {"case_id": "case-001", "passed": true}
96
+ {"case_id": "case-001", "passed": false}
97
+ ```
98
+
99
+ ```
100
+ case_id,passed
101
+ case-001,true
102
+ case-001,false
103
+ ```
104
+
105
+ Fields: `case_id` (a non-empty string), `passed` (a boolean; in CSV one of true,
106
+ false, 1, 0) and, optionally, `trial` (a whole number, on every line or on none;
107
+ each case and trial pair must be unique). Any other field is rejected. File-format
108
+ error messages name the file, line and field, never the value; when `compare` finds
109
+ cases present in only one file, it shows up to three of their ids, cut to 40
110
+ characters. Record failed, crashed or
111
+ timed-out runs as failures: dropping or re-running a trial because of its outcome
112
+ breaks assumption A4 below. Without `trial`, a file concatenated twice cannot be
113
+ detected, so every command prints the number of cases and trials it read; check it.
114
+ `load_outcomes(path)` reads a file into the dictionary the Python API takes.
115
+
116
+ ## Command line
117
+
118
+ ```bash
119
+ python -m realdelta interval results.jsonl # pass rate with its interval
120
+ python -m realdelta compare baseline.jsonl candidate.jsonl # paired change, suite scope
121
+ python -m realdelta plan --cases 40 --delta -0.05 # power per trials option
122
+ python -m realdelta plan --cases 40 --break-cases 2 --trials 3,5,10 \
123
+ --input-tokens 3000 --output-tokens 500 --usd-per-m-input 1 --usd-per-m-output 5
124
+ ```
125
+
126
+ After installing, `realdelta ...` is the same command as `python -m realdelta ...`.
127
+ Add `--json` for one JSON object. Reports from `compare` and `plan` (and
128
+ from `interval` in the population scope) end with notes on what the result cannot
129
+ tell you; warnings (on stderr) flag settings outside the tested range below, such
130
+ as fewer than 10 cases or fewer than 3 trials per case for a comparison. `compare`
131
+ uses seed 0 for its permutation p-value (with more than 16 cases; up to 16 it
132
+ enumerates every sign flip) unless you pass `--seed`.
133
+
134
+ ## Synthetic demo
135
+
136
+ [`examples/synthetic_demo.py`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/examples/synthetic_demo.py) simulates an agent with 40
137
+ cases that mostly pass (case pass rates from Beta(20,1); true pass rate 0.95507), run 5
138
+ times each, with one fixed seed (the first one tried) and no model calls. Its full
139
+ output is in [`examples/synthetic/transcript.txt`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/examples/synthetic/transcript.txt);
140
+ a test re-runs it and compares the output byte for byte. The comparison runs only with
141
+ numpy 2.4.4, the version the transcript was recorded with (other versions skip it).
142
+ The CI workflow pins 2.4.4, and the comparison has passed there on Linux and Windows
143
+ with Python 3.11, 3.12 and 3.13.
144
+
145
+ - **Noise alone moves the number.** Across 500 pairs of runs of the *unchanged*
146
+ agent, the pass rate moved by 2 points or more in 184 pairs (36.8%) and by 4 points
147
+ or more in 35 (7.0%); the 95th percentile of the move is 4.0 points. `compare`
148
+ flagged a change in 23 of the 500 (4.6%); its sign-flip p-value was below 0.05 in
149
+ 14 (2.8%).
150
+ - **A regression in a few cases is caught by the interval much more often than by
151
+ the p-value.**
152
+ In a candidate where 3 of the 40 cases fail every time (true change -0.073), the
153
+ interval excluded 0 in 423 of 500 pairs of runs (84.6%); the sign-flip p-value was
154
+ below 0.05 in 76 (15.2%). The sign-flip test asks whether cases moved in a
155
+ consistent direction across the suite, and only 3 cases moved; the interval
156
+ measures the trial-to-trial noise of this fixed suite, which is the suite scope's
157
+ question. A permutation test within cases would match that scope; it is not
158
+ implemented.
159
+ - **How many trials?** `plan` for suites like this one (40 cases, Beta(20,1), 3 cases
160
+ broken; it draws a new suite in every simulation) gives a simulated power of
161
+ 0.13 +- 0.02 with 1 trial per case (below the gated range), 0.54 +- 0.03 with 3,
162
+ 0.85 +- 0.02 with 5 and 0.99 +- 0.01 with 10.
163
+
164
+ These are simulation results for this setting, not calibration claims.
165
+
166
+ ## Demo on a real model
167
+
168
+ [`examples/real_model/`](https://github.com/althea-yuquan-chen/realdelta/tree/v0.1.0/examples/real_model/) runs 40 generated puzzles (arithmetic, letter
169
+ counting, weekdays, sorting; 10 each) on GPT-5.6 Luna served through Azure OpenAI, with reasoning
170
+ effort low. Azure's content filtering is part of the system measured. Everything that decides the
171
+ result was fixed in [`PROTOCOL.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/examples/real_model/PROTOCOL.md) before any paid call; its
172
+ Amendment 1 changed the model from Claude Haiku 5.5 to GPT-5.6 Luna before any call was made. Three
173
+ arms, 5 trials per case each, 600 calls in one interleaved order: A1 and A2 use the same system prompt
174
+ (an A/A pair), B adds "Answer immediately; do not show your working." The run
175
+ ([`run_20261116/summary.txt`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/examples/real_model/run_20261116/summary.txt)) used 51,213 input and
176
+ 52,408 output tokens with no re-issued calls, timeouts, refusals or truncated outputs. The run
177
+ directory is named after the local date; it started at 2026-11-17T00:54Z.
178
+
179
+ **What "pass" measures here.** A trial passes only if its value is right *and* its last line has the
180
+ form `ANSWER: <value>`, as the pre-registered grader requires. The prompts carry two format
181
+ instructions side by side: the system prompt asks for that line, the user message says "Give the
182
+ answer in this form: ...". Most failures in this run are a correct value without the prefix, so the
183
+ pass rates below mostly measure compliance with the answer format, not puzzle-solving accuracy.
184
+
185
+ | Arm | Pass rate (suite scope) | 95% interval |
186
+ |---|---|---|
187
+ | A1 | 0.875 | [0.821, 0.917] |
188
+ | A2 | 0.855 | [0.798, 0.901] |
189
+ | B | 0.820 | [0.760, 0.871] |
190
+
191
+ - **A1 vs A2:** -0.020, interval [-0.086, +0.046]. The A/A pair was not flagged. It is one
192
+ realisation: in the synthetic demo, 4.6% of unchanged pairs were flagged.
193
+ - **A1 vs B:** -0.055. No change detected at this scale (40 cases, 5 trials); the interval
194
+ [-0.123, +0.014] bounds the change. Given the point above, what this comparison measures is
195
+ mostly format compliance, not accuracy.
196
+ - **Failures per arm** (pass / wrong value / no `ANSWER:` line; no other category occurred):
197
+ A1 175 / 1 / 24, A2 171 / 1 / 28, B 164 / 2 / 34. The grader checks the answer line first, so a
198
+ wrong value without the prefix counts as "no `ANSWER:` line".
199
+ - **Population scope:** because the cases are random draws from the puzzle generator (10 per family;
200
+ treating that stratified sample as independent draws is usually conservative), A1's pass rate for
201
+ puzzles from this generator is 0.875 with interval [0.732, 0.958].
202
+
203
+ Descriptive, not pre-registered, and not tested:
204
+
205
+ - **Value correctness ignoring the prefix:** reading the last line's value with or without `ANSWER:`,
206
+ A1 has 196 of 200 trials right, A2 199 and B 197. 82 of the 86 "no `ANSWER:` line" trials carry the
207
+ right value.
208
+ - **Failures by puzzle family** (any category):
209
+
210
+ | Family | A1 | A2 | B |
211
+ |---|---|---|---|
212
+ | arithmetic | 9 | 6 | 3 |
213
+ | letters | 0 | 5 | 21 |
214
+ | weekday | 15 | 17 | 6 |
215
+ | sort | 1 | 1 | 6 |
216
+
217
+ In this run B failed more on letter counting and sorting and less on weekdays and arithmetic, so
218
+ per-family differences partly cancel in the overall rate. No per-family difference was tested.
219
+ - **Flaky cases:** 15 to 20 cases per arm passed on some trials and failed on others.
220
+
221
+ A1 appears in both comparisons, so they are not independent, and each can raise a false alarm (4.6%
222
+ of unchanged pairs in the synthetic demo; the calibration tests allow up to 7%). The task set is
223
+ synthetic and says nothing about other tasks. Dollar cost is not quoted here because Azure's price
224
+ for this model is uncertain; `summary.txt` gives an upper estimate.
225
+
226
+ ## How the intervals were checked
227
+
228
+ The calibration grid (its settings, the 0.93 gate and the rules for failures) was
229
+ pre-registered in [`docs/calibration_protocol.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/docs/calibration_protocol.md)
230
+ before any final result existed. The suite pass-rate method was replaced after the
231
+ first final run failed; the steps are below. Each grid row simulates suites with
232
+ known case pass rates and counts how often the 95% interval contains the truth.
233
+
234
+ **Claim.** On the grid run [`results/calibration_grid_20261117.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/calibration_grid_20261117.md)
235
+ (2000 repetitions per row, so a Monte Carlo standard error of about 0.005; 95% level),
236
+ every gated row has an estimated coverage of at least 0.93. The lowest rows are
237
+ 0.956 for the suite pass rate (36 rows), 0.951 for the population pass rate (40 rows)
238
+ and 0.945 for the suite delta (25 rows). This is a statement about the tested
239
+ settings, not a guarantee of 95% coverage elsewhere.
240
+
241
+ **Tested settings** (gated rows; only the 95% level):
242
+
243
+ | Quantity | Cases | Trials per case | Case pass rates |
244
+ |---|---|---|---|
245
+ | Suite pass rate | 5 to 50 | 1 to 10, and unequal {1, 3, 10} | Beta(0.5,0.5), (2,2), (4,2), (20,1), (50,1); two mixtures; homogeneous 0.5 and 0.9 |
246
+ | Population pass rate | 5 to 50 | 1 to 10, and unequal {1, 3, 10} | as above, plus a 95%/5% pass/fail mixture and sticky trials (see Limitations) |
247
+ | Suite delta | 10 to 50 | 3 to 10, and unequal {1, 3, 10} at 30 cases | Beta(4,2) and Beta(20,1); no change, +0.05 and +0.10 |
248
+
249
+ Swapping baseline and candidate negates the delta interval exactly (a test pins
250
+ this), so the +0.05 and +0.10 delta rows also cover drops of the same realised size
251
+ from the shifted baseline; they do not cover drops from Beta(4,2) or Beta(20,1)
252
+ themselves, nor a few cases breaking outright.
253
+ The table gives ranges, not a full cross of every combination;
254
+ [`results/calibration_grid_20261117.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/calibration_grid_20261117.md) lists every row.
255
+ File names carry the seed base (20261116, 20261117, 20261118), not a date; both grid
256
+ runs started on 2026-11-16 (see their provenance files).
257
+
258
+ **How we got there.** This history is part of the evidence, so it is listed in full.
259
+
260
+ 1. The first final grid ([`results/calibration_grid_20261116.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/calibration_grid_20261116.md))
261
+ failed one suite row and the population-delta width rule. With 5 cases, 1 trial
262
+ each and rates near 100%, the Wilson interval then used covered 0.917: a single
263
+ failure pulls its upper bound below the truth.
264
+ 2. Amendment 1 pre-registered an exact analytic check (minimum coverage >= 0.947)
265
+ and two candidates: Clopper-Pearson on the effective sample size, and Wilson
266
+ with a continuity correction. Both failed it, at 0.9449 and 0.9434
267
+ ([`results/suite_bounds_selection.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/suite_bounds_selection.md)).
268
+ 3. Amendment 2 added one candidate: Clopper-Pearson with the effective sample size
269
+ capped at the number of trials run. It was designed *after* seeing step 2's
270
+ results, so the analytic check is not independent evidence for it. It passed
271
+ that check with 0.9503 ([`results/suite_bounds_selection_a2.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/suite_bounds_selection_a2.md)).
272
+ The independent evidence is the CI coverage tests, whose gates did not change,
273
+ and the 20261117 grid, which ran once after the implementation.
274
+
275
+ **The cost: wider, often conservative intervals.** Suite pass-rate intervals are
276
+ about 1.11 times as wide as the Wilson intervals of the 20261116 run (median across
277
+ rows, range 1.05 to 1.24; the two runs used different seeds, so the ratio is noisy).
278
+ Seven of the 36 suite pass-rate rows cover more than 99.5% (up to 0.999). With equal
279
+ trials per case the suite interval is Clopper-Pearson on the pooled pass count, which
280
+ is conservative when most cases are deterministic. The population pass rate is more
281
+ conservative still: if all m cases pass, its lower bound is 0.025^(1/m) (0.83 for 20
282
+ cases) however many trials each case ran, because m cases cannot rule out a failure
283
+ mode affecting a few percent of cases like them.
284
+
285
+ ## Limitations
286
+
287
+ - **No population-scope delta in v0.1.** It failed the pre-registered width rule in
288
+ the 20261116 run (mean width 0.368 and 0.364 against a limit of 0.30), so
289
+ `delta_interval(scope="population")`, `compare(scope="population")` and
290
+ `plan_trials(scope="population")` raise `NotImplementedError`.
291
+ - **The suite delta is gated only for 10 to 50 cases with 3 to 10 trials each**
292
+ (plus one unequal-trials setting). It combines the two arms' Wilson-based
293
+ intervals (MOVER), and Wilson bounds under-cover near 100%. A reported-only
294
+ diagnostic ([`results/delta_boundary_diagnostic_20261118.md`](https://github.com/althea-yuquan-chen/realdelta/blob/v0.1.0/results/delta_boundary_diagnostic_20261118.md))
295
+ covers part of the gap near 100%: with case rates from Beta(50,1) or Beta(20,1),
296
+ 5 or 20 cases and 1 or 3 trials each, under no change, a 0.05 drop and (Beta(20,1)
297
+ only) a 0.05 rise, coverage was at least 0.971 across 20 settings. Three cases
298
+ built by hand to be adverse (an all-pass baseline against a candidate just below
299
+ the Wilson bound; not verified to be the worst) reached at least 0.949. Settings
300
+ outside these, such as moderate pass rates with 1 trial per case or fewer than 10
301
+ cases, have no evidence for the delta. `plan_trials` offers 1 trial per case by
302
+ default, which falls in that untested region. Power near 100% is low: a 5-point
303
+ drop is detected in at most 16.8% of runs (20 cases, 3 trials each), and in at most
304
+ 5.1% with 1 trial per case.
305
+ - **The pass-rate and delta intervals use different bounds** (Clopper-Pearson and
306
+ Wilson), so one cannot be derived from the other on the same data.
307
+ - **Unequal trial counts** are covered by three grid settings (two pass-rate, one
308
+ delta) and one exact, reported-only suite pass-rate configuration (5 cases with
309
+ 1, 1, 3, 3 and 10 trials, homogeneous rate: minimum coverage 0.958).
310
+ - **Assumptions.** A1: trials within a case are independent and identically
311
+ distributed. A2: cases are an independent draw from a population. A3: the two arms
312
+ share no randomness (seeds, caches, outage windows). A4: the number of trials per
313
+ case does not depend on outcomes. The suite scope needs A1, A3 and A4; the
314
+ population pass rate needs A2 and A4. The grid includes violations, reported and
315
+ not gated: trials that repeat the previous outcome (violates A1) drop suite
316
+ pass-rate coverage to 0.871 and 0.734; a shared per-trial shock in each arm
317
+ (violates A3) drops suite delta coverage to 0.894, with a false-alarm rate of
318
+ 0.105. A2 holds only if your cases were sampled from the population you care
319
+ about; for a hand-picked suite, the population scope describes a hypothetical
320
+ population of similar cases.
321
+ - **Not checked:** confidence levels other than 95%, and the calibration of the
322
+ p-value from `compare`. It tests the sharp null "the two versions behave
323
+ identically on every case", and in the synthetic demo's setting (3 of 40 cases
324
+ broken) it rejected in 15.2% of runs against 84.6% for the interval.
325
+
326
+ ## Reproduce
327
+
328
+ After installing from a clone (see Install), from the repository root:
329
+
330
+ ```bash
331
+ python -m pytest # unit, analytic and CI coverage tests
332
+ python scripts/calibration_grid.py --smoke # 50 repetitions; not evidence
333
+ ```
334
+
335
+ Both final grid runs are committed in `results/` with a provenance file, and each
336
+ ran once; the diagnostic records its commit in its own header. The grid script
337
+ refuses to overwrite existing results, and in this release it uses seed base
338
+ 20261117; the 20261116 run is reproducible from commit da182ba. The selection and diagnostic scripts are `scripts/suite_bounds_selection.py`
339
+ and `scripts/delta_boundary_diagnostic.py`.
340
+
341
+ ## License
342
+
343
+ Apache-2.0