equiframe-benchmark 0.12.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. equiframe_benchmark/PROTOCOL_V2.md +172 -0
  2. equiframe_benchmark/__init__.py +284 -0
  3. equiframe_benchmark/__main__.py +3 -0
  4. equiframe_benchmark/_scale_workload.py +375 -0
  5. equiframe_benchmark/calibration_types.py +112 -0
  6. equiframe_benchmark/cli.py +196 -0
  7. equiframe_benchmark/commands.py +149 -0
  8. equiframe_benchmark/commands_scale.py +38 -0
  9. equiframe_benchmark/commands_step10.py +275 -0
  10. equiframe_benchmark/commands_v2.py +206 -0
  11. equiframe_benchmark/cost_table.py +62 -0
  12. equiframe_benchmark/costs.py +788 -0
  13. equiframe_benchmark/distribution.py +932 -0
  14. equiframe_benchmark/evaluation.py +614 -0
  15. equiframe_benchmark/evaluation_v2.py +428 -0
  16. equiframe_benchmark/fixtures.py +299 -0
  17. equiframe_benchmark/freeze.py +869 -0
  18. equiframe_benchmark/freeze_v2.py +412 -0
  19. equiframe_benchmark/integration.py +655 -0
  20. equiframe_benchmark/io.py +29 -0
  21. equiframe_benchmark/models.py +313 -0
  22. equiframe_benchmark/models_v2.py +13 -0
  23. equiframe_benchmark/mutations.py +34 -0
  24. equiframe_benchmark/oracle.py +264 -0
  25. equiframe_benchmark/pairwise.py +248 -0
  26. equiframe_benchmark/pilot.py +254 -0
  27. equiframe_benchmark/py.typed +0 -0
  28. equiframe_benchmark/reconciliation.py +294 -0
  29. equiframe_benchmark/reproduction.py +542 -0
  30. equiframe_benchmark/scale_io.py +19 -0
  31. equiframe_benchmark/scale_process.py +1077 -0
  32. equiframe_benchmark/scale_types.py +120 -0
  33. equiframe_benchmark/scaling.py +529 -0
  34. equiframe_benchmark/schemas/protocol-v2/CommonSupportV2-2.0.schema.json +327 -0
  35. equiframe_benchmark/schemas/protocol-v2/ContrastSpecV2-2.0.schema.json +37 -0
  36. equiframe_benchmark/schemas/protocol-v2/EvaluationConfigV2-2.0.schema.json +184 -0
  37. equiframe_benchmark/schemas/protocol-v2/EvaluationReportV2-2.0.schema.json +1237 -0
  38. equiframe_benchmark/schemas/protocol-v2/EvaluationStratumV2-2.0.schema.json +747 -0
  39. equiframe_benchmark/schemas/protocol-v2/FreezeDefinitionV2-2.0.schema.json +5951 -0
  40. equiframe_benchmark/schemas/protocol-v2/FreezeManifestV2-2.0.schema.json +6222 -0
  41. equiframe_benchmark/schemas/protocol-v2/PrincipalContrastV2-2.0.schema.json +578 -0
  42. equiframe_benchmark/schemas/protocol-v2/ProtocolReviewV2-2.0.schema.json +56 -0
  43. equiframe_benchmark/schemas/protocol-v2/SignedMetricV2-2.0.schema.json +51 -0
  44. equiframe_benchmark/schemas/protocol-v2/StudyCaseResultV2-2.0.schema.json +4881 -0
  45. equiframe_benchmark/schemas/protocol-v2/StudyResultV2-2.0.schema.json +8370 -0
  46. equiframe_benchmark/schemas/protocol-v2/TableExportV2-2.0.schema.json +75 -0
  47. equiframe_benchmark/selection.py +178 -0
  48. equiframe_benchmark/snapshot.py +24 -0
  49. equiframe_benchmark/splits.py +92 -0
  50. equiframe_benchmark/study.py +526 -0
  51. equiframe_benchmark/study_bundle.py +379 -0
  52. equiframe_benchmark/study_bundle_v2.py +388 -0
  53. equiframe_benchmark/study_v2.py +544 -0
  54. equiframe_benchmark/tables.py +347 -0
  55. equiframe_benchmark/tables_v2.py +340 -0
  56. equiframe_benchmark/training_types.py +52 -0
  57. equiframe_benchmark-0.12.4.dist-info/METADATA +343 -0
  58. equiframe_benchmark-0.12.4.dist-info/RECORD +62 -0
  59. equiframe_benchmark-0.12.4.dist-info/WHEEL +4 -0
  60. equiframe_benchmark-0.12.4.dist-info/entry_points.txt +3 -0
  61. equiframe_benchmark-0.12.4.dist-info/licenses/LICENSE +21 -0
  62. equiframe_benchmark-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md +8 -0
@@ -0,0 +1,52 @@
1
+ """Selection audit stays in the benchmark; only vectors cross into the library."""
2
+
3
+ from equiframe import NormalTrainingProfiles
4
+ from pydantic import model_validator
5
+ from typing_extensions import Self
6
+
7
+ from equiframe_benchmark.models import Digest, Document, Text, digest
8
+
9
+
10
+ class TrainingRecord(Document):
11
+ scenario_id: Text
12
+ candidate_id: Text
13
+ group_id: Digest
14
+ candidate_sha256: Digest
15
+ oracle_sha256: Digest
16
+ profile_sha256: Digest
17
+ vector_sha256: Digest
18
+ execution_sha256: Digest
19
+
20
+
21
+ class TrainingManifest(Document):
22
+ purpose: str = "normal_training"
23
+ catalog_sha256: Digest
24
+ splits_sha256: Digest
25
+ records: tuple[TrainingRecord, ...]
26
+
27
+ @model_validator(mode="after")
28
+ def unique(self) -> Self:
29
+ if self.purpose != "normal_training" or not self.records:
30
+ raise ValueError("nonempty normal training selection required")
31
+ for field in ("candidate_id", "execution_sha256", "profile_sha256"):
32
+ if len({getattr(r, field) for r in self.records}) != len(self.records):
33
+ raise ValueError("duplicate training evidence")
34
+ return self
35
+
36
+
37
+ class TrainingSelection(Document):
38
+ manifest: TrainingManifest
39
+ training: NormalTrainingProfiles
40
+
41
+ @model_validator(mode="after")
42
+ def consistent(self) -> Self:
43
+ if digest(self.manifest) != self.training.selection_manifest_sha256:
44
+ raise ValueError("selection manifest mismatch")
45
+ if len(self.manifest.records) != len(self.training.profiles):
46
+ raise ValueError("selection size mismatch")
47
+ if any(
48
+ record.vector_sha256 != digest(vector)
49
+ for record, vector in zip(self.manifest.records, self.training.profiles, strict=True)
50
+ ):
51
+ raise ValueError("selected vectors do not match the recorded evidence")
52
+ return self
@@ -0,0 +1,343 @@
1
+ Metadata-Version: 2.4
2
+ Name: equiframe-benchmark
3
+ Version: 0.12.4
4
+ Summary: EquiFrame Verify, paired evaluation and reproducible experimental evidence
5
+ Project-URL: Homepage, https://github.com/htroya/equiframe
6
+ Project-URL: Source, https://github.com/htroya/equiframe
7
+ Project-URL: Documentation, https://github.com/htroya/equiframe/blob/main/docs/index.md
8
+ Project-URL: Issue Tracker, https://github.com/htroya/equiframe/issues
9
+ Author: Hernán Troya
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ License-File: THIRD_PARTY_NOTICES.md
13
+ Keywords: benchmark,data-validation,dataframes,paired-evaluation,reproducibility
14
+ Classifier: Development Status :: 4 - Beta
15
+ Classifier: Operating System :: Microsoft :: Windows
16
+ Classifier: Operating System :: POSIX :: Linux
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: <3.14,>=3.10
24
+ Requires-Dist: equiframe[arrow]==0.12.4
25
+ Requires-Dist: packaging<27,>=26
26
+ Requires-Dist: tomli<3,>=2.2; python_version < '3.11'
27
+ Requires-Dist: typing-extensions<5,>=4.14
28
+ Provides-Extra: engines
29
+ Requires-Dist: equiframe[engines]==0.12.4; extra == 'engines'
30
+ Provides-Extra: ml
31
+ Requires-Dist: equiframe[ml]==0.12.4; extra == 'ml'
32
+ Provides-Extra: resources
33
+ Requires-Dist: psutil==7.2.2; extra == 'resources'
34
+ Description-Content-Type: text/markdown
35
+
36
+ # EquiFrame Benchmark 0.12.4
37
+
38
+ > Release target: **0.12.4**. Check the official index for availability and
39
+ > verify the identified artifact hashes. Publication requires artifact-bound
40
+ > release checks; prior candidate results keep their own identities.
41
+
42
+ **Experimental evaluation, reproducible evidence and bounded engineering costs.**
43
+
44
+ [Principal paired protocol V2](PROTOCOL_V2.md) provides a separately versioned
45
+ freeze → native study → paired evaluation → seven tables → portable reproduction
46
+ route for the three primary families and three engine pairs. V1 readers,
47
+ fingerprints and common-support results remain unchanged. Earlier protocol
48
+ work included twenty-four focused engineering checks, with API/CLI roundtrips
49
+ over fictional held-out native cases and portable reproduction. Those results
50
+ retain their original source and artifact identities; they do not establish
51
+ 0.12.4 validation or scientific approval. Final protocol review remains required.
52
+
53
+ `equiframe-benchmark` is separate from the core `equiframe` package. It provides
54
+ scenario catalogs, independent-review records, grouped splits, reviewed normal
55
+ selection, frozen studies, paired evaluation, reproducible tables/bundles and
56
+ resource measurements. The core never imports it or receives oracle labels as
57
+ detector features.
58
+
59
+ ## Release scope and compatibility
60
+
61
+ This distribution is paired exactly with `equiframe[arrow]==0.12.4` and
62
+ supports Python `>=3.10,<3.14`. Original code is licensed under [MIT](LICENSE);
63
+ dependencies retain their own terms in [third-party notices](THIRD_PARTY_NOTICES.md).
64
+
65
+ The owner authorized publication on 2026-10-27 and confirmed local manual
66
+ publication on 2026-10-28. Uploads use an ephemeral local API token, revoked
67
+ after use; no GitHub Actions job or stored GitHub publishing secret is used.
68
+ The release checklist binds the inspected commit and all four artifact hashes,
69
+ checks the security review and clean consumers, and promotes the same bytes.
70
+ A version label or this README does not establish a successful upload.
71
+
72
+ The benchmark's scientific protocols and APIs remain experimental. Software
73
+ correctness tests preserve their documented contracts but do not establish
74
+ real-data accuracy, reviewed corpus labels, independent human acceptance,
75
+ external adoption or a deployed service/data warehouse.
76
+
77
+ Hernán Troya (`htroya`) is the confirmed maintenance, release and security owner,
78
+ with no response-time SLA. Documentation and reporting routes share the
79
+ [EquiFrame repository](https://github.com/htroya/equiframe).
80
+ Sensitive reports use its
81
+ [private reporting channel](https://github.com/htroya/equiframe/security/advisories/new).
82
+ For each release, the maintainer must enable public issues and private reporting
83
+ and verify their availability.
84
+ Historical Alpha artifacts and results retain their original identity and scope.
85
+
86
+ The core wheel includes the self-contained English handbook with API, semantics,
87
+ compatibility, contribution, security, dependency/SBOM and release procedures:
88
+
89
+ ```python
90
+ from importlib.resources import files
91
+
92
+ print(files("equiframe").joinpath("docs/index.md").read_text(encoding="utf-8"))
93
+ ```
94
+
95
+ From an authorized monorepo checkout the same guide is at
96
+ [core handbook](https://github.com/htroya/equiframe/blob/main/docs/index.md); installed consumers do not need
97
+ repository access. This README is included in distribution metadata. Benchmark
98
+ sdists also include examples, tests, tools and JSON schemas.
99
+
100
+ ## Install exact artifacts, then run outside the checkout
101
+
102
+ Obtain both identified wheels and a reviewed dependency wheelhouse for your
103
+ interpreter/platform/profile. Verify supplied SHA-256 values before installation.
104
+ In a fresh directory outside the repository, using a POSIX shell:
105
+
106
+ ```bash
107
+ python3.12 -m venv .venv
108
+ . .venv/bin/activate
109
+ CORE=/absolute/path/equiframe-0.12.4-py3-none-any.whl
110
+ BENCH=/absolute/path/equiframe_benchmark-0.12.4-py3-none-any.whl
111
+ WHEELHOUSE=/absolute/path/reviewed-wheelhouse
112
+ python -m pip install --no-index --find-links "$WHEELHOUSE" "${CORE}[engines]" "${BENCH}[resources]"
113
+ python -m pip check
114
+ python -I -c "import equiframe, equiframe_benchmark; print(equiframe.__file__); print(equiframe_benchmark.__file__)"
115
+ python -I -m equiframe_benchmark.pilot --output pilot-synthetic
116
+ ```
117
+
118
+ Absolute paths are placeholders for verified local files. Missing compatible
119
+ binary dependencies are a blocker; do not drop hashes or switch to an unrelated
120
+ index package. Avoid editable installs/PYTHONPATH as artifact validation. Windows
121
+ can use `py -3.12 -m venv .venv` and `.venv\Scripts\python.exe` directly without
122
+ changing activation security policy. Command conventions are not platform tests.
123
+
124
+ The benchmark base requires Arrow through core and `packaging>=26,<27`.
125
+ `engines` adds pandas/Polars/DuckDB; `ml` adds the core ML extra;
126
+ `resources` adds `psutil==7.2.2`. Choose reviewed dependency pins for the required
127
+ profiles. Metadata admission does not prove all platforms or range combinations.
128
+ Historical measured cells are Linux x86_64/Python 3.12.14 and 3.13.5; a current
129
+ candidate needs its own exact-artifact receipt. Manual CI configuration is not a
130
+ remote job result, and Windows/macOS/ARM64 are not newly claimed as verified.
131
+
132
+ The pilot destination must be new. `equiframe-benchmark-pilot --output DIR` is an
133
+ installed equivalent entry point. The synthetic pilot produces 5 scenarios,
134
+ 35 candidates, 15 profiles and 4 dependency groups. **All 35 reviews remain
135
+ pending**. Its 15 mutations are artifact injections; its 15 control
136
+ transformations execute natively. These are not a completed thesis sample or
137
+ trained scientific model.
138
+
139
+ ## Review, grouping and normal selection
140
+
141
+ Public models include `Scenario`, `Catalog`, `Candidate`, `ReviewPacket`,
142
+ `ReviewAssignment`, `Review`, `OracleResult`, `SplitManifest`, `TrainingSelection`
143
+ and `CalibrationSelection`. `load_document`, `save_document` and `digest` use
144
+ strict canonical JSON and explicit evidence identity.
145
+
146
+ This snippet continues from the synthetic pilot above:
147
+
148
+ ```python
149
+ from pathlib import Path
150
+ import equiframe_benchmark as eb
151
+
152
+ root = Path("pilot-synthetic")
153
+ catalog = eb.load_document(eb.Catalog, root / "catalog.json")
154
+ splits = eb.load_document(eb.SplitManifest, root / "splits.json")
155
+ eb.validate_splits(catalog, splits)
156
+ print(eb.split_counts(catalog, splits))
157
+ scenario = catalog.scenarios[0]
158
+ candidate = eb.load_document(eb.Candidate, root / "nulls_types" / "pandas-candidate.json")
159
+ packet = eb.review_packet(scenario, candidate, directory=Path("review-packet"))
160
+ pending = eb.evaluate_oracle(scenario, candidate, packet=packet, work_dir=Path("oracle-evidence"))
161
+ assert pending.status == "pending" and pending.label is None
162
+ ```
163
+
164
+ Only the blinded packet goes to the reviewer. Keep catalog/candidate metadata,
165
+ alerts and other reviewers' decisions separate. Record two independent reviewers
166
+ and an arbiter in an assignment before review, with UTC timestamps. Reviews
167
+ require packet hash, identity declaration, decision, rationale, timestamp and
168
+ blind attestation. Resolution is permitted for disagreement by the assigned
169
+ arbiter. Do not generate human approvals from comparator output. Hashes and JSON
170
+ names do not authenticate a human or establish true independence.
171
+
172
+ `dependency_groups` closes transitive declared/artifact dependencies before
173
+ `make_splits`; `validate_splits` and `split_counts` retain group/partition scope.
174
+ `select_normal_training` and `select_normal_calibration` require candidate,
175
+ oracle, execution and profile maps with exact linked identities, one family and
176
+ feature definition, confirmed normal O, and the appropriate disjoint partition.
177
+ The pending pilot is not eligible normal training evidence.
178
+
179
+ `audit_detector_selections` checks shared catalog/split, compatible features,
180
+ independent groups and no reused scenarios/executions/profiles. Respect
181
+ `eligible`/`not_estimable`, preserve counts and justify minimum support before
182
+ freeze. Core `fit_detectors`, `score_detectors`, `calibrate_detectors`, `detect`
183
+ and `detect_component` separate fitting, scoring, calibration and inference.
184
+ No anomaly/no-alert state is a comparison oracle; test never fits the model.
185
+
186
+ ## Public workflow reference
187
+
188
+ The exhaustive experimental export namespace is `equiframe_benchmark.__all__`.
189
+ Use `inspect.signature(eb.function)` and each model's `model_json_schema()` for
190
+ installed signatures/defaults. These are distinct families:
191
+
192
+ | Family | Entry points |
193
+ |---|---|
194
+ | Blinded review | `review_packet`, `evaluate_oracle`, `oracle_verification_bridge` |
195
+ | Dependency splits | `dependency_groups`, `make_splits`, `validate_splits`, `split_counts` |
196
+ | Reviewed selections | `select_normal_training`, `select_normal_calibration`, `audit_detector_selections` |
197
+ | Snapshot consumer | Compatibility aliases of core `SnapshotClient`, `SnapshotDataSource`, `snapshot_fingerprint` |
198
+ | Three-engine integration | `run_integration`, `read_integration` |
199
+ | Domain-configurable reconciliation | `reconcile_awards`, `AwardReconciliationSpec` |
200
+ | Freeze/study | `audit_freeze`, `create_freeze`, `read_freeze`, `run_study`, `read_study` |
201
+ | Evaluation | `evaluate_records`, `evaluate_pairwise`, `evaluation_protocol_sha256` |
202
+ | Tables | `table_payloads`, `write_tables`, `read_tables` |
203
+ | Scientific cost protocol | `measure_costs`, `read_costs`, `write_cost_table` |
204
+ | Record reproduction | `make_reproduction_record`, `compare_reproductions` |
205
+ | Generic bundles | `create_bundle`, `read_bundle`, `bundle_artifact`, `reproduce_bundle` |
206
+ | Study bundles | `bundle_study`, `regenerate_study_record`, `reproduce_study_bundle` |
207
+ | Engineering scale | `measure_scale`, `read_scale`, `summarize_scale` |
208
+ | Distribution inventory | `inspect_distribution`, `DistributionReadiness`, `NameCheck` |
209
+
210
+ Snapshot consumers pin a declared publication and verify artifacts from an
211
+ explicit trusted client. `run_integration` executes each of three registered
212
+ native implementations once, then compares three pairs. A separate optional
213
+ expected output is fixed before execution. Shared captures do not become
214
+ additional independent observations when reused in several pair comparisons.
215
+ If a native execution fails, its expected-output comparison is recorded as
216
+ `execution_unavailable`; the operation stays incomplete and healthy independent
217
+ pairs retain their evidence. The synthetic date fixture uses Pandas' native UTC
218
+ microsecond dtype before extracting date32, including empty and NULL values,
219
+ so it does not require Arrow's separate C++ timezone database on Windows.
220
+ Client/DW authentication, service writer coordination and production acceptance
221
+ are not implemented by the consumer interface.
222
+
223
+ ## Freeze, study and reproduction
224
+
225
+ `FreezeDefinition` records catalog, grouped splits, cases, family model slots,
226
+ reviewed selections, prior decisions, files/hashes, environment and evaluation/cost
227
+ protocols. Pilot freeze verifies technical readiness while retaining scientific
228
+ blockers. Final freeze requires those blockers resolved; it does not manufacture
229
+ reviews or fit on test data.
230
+
231
+ The following are command **templates** using your own reviewed files and trusted
232
+ registry factory; they are not a complete runnable scientific experiment:
233
+
234
+ ```text
235
+ equiframe-benchmark freeze definition.json --output freeze.json
236
+ equiframe-benchmark verify-freeze freeze.json
237
+ equiframe-benchmark run-study freeze.json --registry my_etl:create_registry --output studies
238
+ equiframe-benchmark verify-study studies/study_ID
239
+ equiframe-benchmark evaluate studies/study_ID --output evaluation.json
240
+ equiframe-benchmark tables studies/study_ID --output regenerated-tables
241
+ equiframe-benchmark bundle-study studies/study_ID --output portable-study
242
+ equiframe-benchmark reproduce-study portable-study --expected-sha256 TRUSTED_BUNDLE_MANIFEST_SHA256
243
+ ```
244
+
245
+ Factory references import and execute trusted code explicitly. JSON requests do
246
+ not authorize arbitrary plugins or contain credentials. The frozen historical
247
+ main comparison uses confirmed O and common complete R/M/B/H support, retaining
248
+ individual support, partial alerts, failures and exclusions. `evaluate_pairwise`
249
+ adds six descriptive pairwise supports with their own denominators; it preserves
250
+ the original report/protocol/six CSV and remains `scientific=False`. Using that
251
+ analysis as a new scientific primary protocol requires a new freeze.
252
+
253
+ Undefined precision, recall, F1 and FPR remain absent with reasons. Average
254
+ precision requires two classes and a common family/model. Bootstrap samples
255
+ whole groups and requires a prior independence rationale. CSVs regenerate from
256
+ verified records; editing a CSV and recomputing its hash does not create valid
257
+ changed evidence.
258
+
259
+ Study bundles preserve original captures/manifests and regenerate configuration,
260
+ splits, labels, decisions and tables without rerunning ETL. They do not certify a
261
+ colleague independently repeated the experiment. Raw costs are retained without
262
+ requiring identical timings on reproduction. The distribution inventory does not
263
+ reserve a name, resolve licensing, install a package or authorize publication.
264
+
265
+ ## Cost protocols and engineering scale
266
+
267
+ `measure_costs` uses a frozen scientific `CostProtocol`: isolated profile+M versus
268
+ full comparison over already captured outputs, one warmup and five alternating
269
+ repetitions per method. Initial ETL/capture is excluded. `cost-table` retains all
270
+ twelve raw observations, warmups, failures and reasons. Repetitions do not increase
271
+ the independent detection sample. Missing monitoring is never imputed as zero.
272
+
273
+ The separate engineering protocol `ScaleSpec`/`ScaleTrial`/`ScaleRun` measures
274
+ bounded synthetic integer/text/NULL workloads. See the installed core handbook's
275
+ `docs/performance.md` for a complete small script, stage boundaries and budgets.
276
+ Source distributions also contain `examples/scale_local.py`, which defaults to
277
+ 1k rows; 100k/1M require explicit `--rows` and larger recorded budgets.
278
+
279
+ One isolated trusted worker records fresh/warm process groups with uncontrolled
280
+ OS caches, nine stages, sampled RSS/CPU/I/O, disk and evidence bytes. RSS/disk
281
+ thresholds are reactive and can overshoot; they are not hard quotas. A valid
282
+ completion marker, expected verdict and verified inventory are required.
283
+ `complete` and `resource_measurement_complete` are separate: a successful
284
+ operation can have incomplete observations.
285
+
286
+ Restricted native pandas/DuckDB baselines provide less evidence; time ratios
287
+ are not equivalent-service speed rankings. DataComPy is unpinned/unavailable,
288
+ with no invented timing. Kill, missing finalization, failed/partial measurement,
289
+ ENOSPC and last-batch differences stay visible. Receipts are location-bound;
290
+ reproduction uses the specification and exact wheels, not an implied portable run.
291
+
292
+ ## CLI exits and safe interpretation
293
+
294
+ `equiframe-benchmark --help` lists the installed commands, including catalog
295
+ `validate`, `split`, `pilot`, integration, freeze/study, evaluation, costs and scale.
296
+
297
+ - Study/integration operations distinguish complete without alert (0), complete
298
+ with alert/difference (1), invalid input/freeze (2) and partial/unavailable (3)
299
+ - Integrity `verify-*` commands can return 0 for intact evidence of a historical
300
+ incomplete run; read the stored semantic state. They do not turn it into success
301
+ - Reproduction disagreement uses 1; invalid evidence uses a failure code
302
+ - `costs` uses 3 for incomplete measurement
303
+ - `scale` and `verify-scale` use 3 for unsuccessful/incomplete measurement or
304
+ invalid finalization. Their zero is expected-workload completion, not equivalence
305
+
306
+ The historical core `equiframe.benchmark` placeholder is quarantined and not a
307
+ compatible shortcut into this package. Do not reenable its removed metrics or
308
+ infer scientific results from synthetic demonstrations.
309
+
310
+ ## Development and contribution
311
+
312
+ Operational errors retain their original first stderr line and exit status,
313
+ then add one fixed-text `hint[category]: ...` line. For example, an existing
314
+ output recommends a new path instead of overwriting evidence; a missing file
315
+ recommends checking the input and parent directory. Hints omit raw exception
316
+ messages, data and paths. Automation should read the exit status and first
317
+ line, not compare the complete stderr text. `scale` retains exit 3 for its
318
+ failed operation, including invalid specifications.
319
+ The standalone `equiframe-benchmark-pilot` entrypoint also uses fixed hints;
320
+ its operation failures retain exit 1 and print `pilot_operation_failed`.
321
+
322
+ From `packages/equiframe-benchmark`, with the matching core wheel already built:
323
+
324
+ ```bash
325
+ uv sync --locked --all-extras --reinstall-package equiframe
326
+ uv run --locked --all-extras ruff check .
327
+ uv run --locked --all-extras ruff format --check .
328
+ uv run --locked --all-extras mypy
329
+ uv run --locked --all-extras python tools/generate_assets.py --check
330
+ uv run --locked --all-extras pytest -q
331
+ uv run --locked --all-extras python -m build --no-isolation
332
+ uv run --locked --all-extras twine check --strict dist/*.whl dist/*.tar.gz
333
+ uv run --locked --all-extras python tools/verify_distribution.py
334
+ ```
335
+
336
+ `--reinstall-package equiframe` prevents stale local dependency caching while a
337
+ package version is unchanged. From this directory, the byte-bound candidate driver is
338
+ `python ../../tools/check_candidate.py --package . --output /absolute/path/new-checks.json`.
339
+ These are checks to execute, not claims they already passed this artifact.
340
+ Source tests, exact installed tests and remote CI are distinct evidence. Preserve
341
+ historical fixtures, safe failures and methodological boundaries. Follow the core
342
+ handbook's contribution, upgrade, security and release procedures. No public
343
+ publication or production deployment is authorized by building these artifacts.
@@ -0,0 +1,62 @@
1
+ equiframe_benchmark/__init__.py,sha256=CV76hypwD3EqlNOt31h3H4hy4fWZEG3G4DHad_251YI,7253
2
+ equiframe_benchmark/__main__.py,sha256=pleKgOthX0VewbdCKlEd9TkzE0aImbhDAp8zvJbEroc,49
3
+ equiframe_benchmark/_scale_workload.py,sha256=Yu8ihfuqmh1Qc_VlgGBRlg5P6NW9Jkkji5Ka-VE1970,13731
4
+ equiframe_benchmark/calibration_types.py,sha256=r-sfBQrX9ZuhKOSCdGz2jIKYDekjX8Y7sh8czueg8rA,4534
5
+ equiframe_benchmark/cli.py,sha256=q8GBPyF6LO2uX6cFWMFgemDa5wkyypxn2oqLJPRw9x4,6913
6
+ equiframe_benchmark/commands.py,sha256=qoLAnIwV3u1RrAxfPCwFnyKkbDrMu7IIXcO3vuTEFs8,5619
7
+ equiframe_benchmark/commands_scale.py,sha256=6B8OhI7oAYr2EyzPRsCGmIbV_Q5IB4JXVkl0KXUuKKM,1526
8
+ equiframe_benchmark/commands_step10.py,sha256=-j8moHHnMTPXW-9cJLEPvjmEJzqVDjzglxgvyYS5WIQ,10309
9
+ equiframe_benchmark/commands_v2.py,sha256=ID6Za42JrzdCEYQEs8F6_LWvvievOVcNeCU-L5tqLt8,7917
10
+ equiframe_benchmark/cost_table.py,sha256=oVIBDKq-RKPm3kb8OrhPS6JPqldXopzrhtMRhccB7SY,2025
11
+ equiframe_benchmark/costs.py,sha256=575dRSSfEiS7hkI3RJvOQTw6Y4daIUU1REPBdPkooyU,32590
12
+ equiframe_benchmark/distribution.py,sha256=_NqIyo_zHQe7xUWTH3GZYGQf3rEn5ZSeb_WkJrF0TiA,39689
13
+ equiframe_benchmark/evaluation.py,sha256=4NXMM-wpDRW9gz4_6A8fGzII_TNzguRLrPilRmfQloo,24475
14
+ equiframe_benchmark/evaluation_v2.py,sha256=EshYcCXDv7ld5Aqv-0BtaK0LgZIpdpIkcStEepGZaWM,16083
15
+ equiframe_benchmark/fixtures.py,sha256=RZOhvPUW6gZK5IypHRJdQIdYg8orK5o5dgkZLTgF2UE,11306
16
+ equiframe_benchmark/freeze.py,sha256=_O80pzC8-FuKFSExF0UHAIkWz04TkefiMNcqmoE8E8w,37311
17
+ equiframe_benchmark/freeze_v2.py,sha256=N2r0CFPVSS1BuWyjU6JWoUdVB2S6gMY-dXLb4c30dak,17984
18
+ equiframe_benchmark/integration.py,sha256=xovO12X193SuKFt5vfnQDxx_a-AVaI1DH9XV2NiRiW0,28811
19
+ equiframe_benchmark/io.py,sha256=M53n9ZJwCxhgw58nEunGQAA-lqoG9HBkV3knRU56UMo,845
20
+ equiframe_benchmark/models.py,sha256=SPDcHfIImoX4V0amNvD7QzfY5PU38PFqn8uZJZlPeWY,12262
21
+ equiframe_benchmark/models_v2.py,sha256=vJNdYhdWS_RAw-IPBTDwHp1_eeiwk3feV0lZD1KuXZs,386
22
+ equiframe_benchmark/mutations.py,sha256=W6EhzTq7KqCn4iQUoD0Vc3YEYDjmxruQl6oGbvSrSz8,1195
23
+ equiframe_benchmark/oracle.py,sha256=opDtIa9YKYY1SONdotnh-BuD6Smm7eTelzanMyCoYzY,10495
24
+ equiframe_benchmark/pairwise.py,sha256=kIN6ooxdWezpGu02FSSaoGAaylvF3Pnb--31O5-_lFw,9606
25
+ equiframe_benchmark/pilot.py,sha256=emTNlfUOh0Q77lZchpwG4ufGRmlTfdy94DUNdwAc3vs,10238
26
+ equiframe_benchmark/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
27
+ equiframe_benchmark/reconciliation.py,sha256=3KlXk1PMbzOev9yZTnlGplqTvOQ5I0WIxHkZrxGXmc4,12452
28
+ equiframe_benchmark/reproduction.py,sha256=anpP5nVZpj_OLPFMrsa0fgI0kY9zKNOLw_0qEUtyLtY,23477
29
+ equiframe_benchmark/scale_io.py,sha256=Bn5rzwzvTpMp7y1srLBVNQ0mgRNk71v76Zpd_UkujmQ,719
30
+ equiframe_benchmark/scale_process.py,sha256=IBn1xndN1kfeMExAe8k156QiR8TTuhjQPGogoK-aHAM,45156
31
+ equiframe_benchmark/scale_types.py,sha256=FxCeSysFVjYHf34XCr2b3x-996YX-OcBjkpcd3B-od4,5076
32
+ equiframe_benchmark/scaling.py,sha256=a8x5ArKft7y2j11qYD6r8mSDgxFiQm8bFj7YrSvCTOk,22246
33
+ equiframe_benchmark/selection.py,sha256=ftpQIoZoi-rCQp4YxuPw0GHi7sByeCeyBWtpC2eLtcA,7819
34
+ equiframe_benchmark/snapshot.py,sha256=ysbnD3qoOaIXZt4ZgOacUuEUDibFvHOQHerCim5DOsQ,689
35
+ equiframe_benchmark/splits.py,sha256=a0mBDF_45jeFSr1xFyxGwz7jTLcBsbM5kEN3ydNvTDg,4009
36
+ equiframe_benchmark/study.py,sha256=oSSrJXJR5Y49-L38yQzcrfn2se0YhNp9kkrDVNOPU7E,22154
37
+ equiframe_benchmark/study_bundle.py,sha256=bscyovidROBcC5spnLuBCRuFyLeBybDhjiala_HIO8Y,16741
38
+ equiframe_benchmark/study_bundle_v2.py,sha256=xw_hH052cjMJlnPXx0c8ZhB_Vf1-Q33YzaYnYYbWHfQ,17207
39
+ equiframe_benchmark/study_v2.py,sha256=GM78b2yo1e9zOZPP4HOBY_O9YvFuMma68evJWPdOYL0,22825
40
+ equiframe_benchmark/tables.py,sha256=ELggQg_9_a60lFx_5Bvezp_UdnDQzC5I9CDrmRbL2RM,12287
41
+ equiframe_benchmark/tables_v2.py,sha256=kA27TGX9wd3ZYKvr5d9oxd8ZadF02lQFe1D5OKWmmmg,12158
42
+ equiframe_benchmark/training_types.py,sha256=xcSYoszPl9XLYR-t_1rjpcyAMekdMoWMUriH7C6YLX0,1846
43
+ equiframe_benchmark/PROTOCOL_V2.md,sha256=pzIzYh17wp_5YIVh_6AFHFrelr6uowGpHRWUSRRRZ-4,8787
44
+ equiframe_benchmark/schemas/protocol-v2/CommonSupportV2-2.0.schema.json,sha256=JxsDFunP9_trEHjVYw3mmr2CMxAYOSfFls3C_wODcNY,7084
45
+ equiframe_benchmark/schemas/protocol-v2/ContrastSpecV2-2.0.schema.json,sha256=sovqKGnKDMUugNXPmshZ90JrIsm0uLMwYP-HHgQXSBM,573
46
+ equiframe_benchmark/schemas/protocol-v2/EvaluationConfigV2-2.0.schema.json,sha256=aNDW-LJT8Rg7lfQjuPdJQ-T0UgpZucXgzbCiMstueAI,3907
47
+ equiframe_benchmark/schemas/protocol-v2/EvaluationReportV2-2.0.schema.json,sha256=pXYsEmmQ89PgaKVRjdvI5VhnMzMt0MkCGhwErqy9z-w,28412
48
+ equiframe_benchmark/schemas/protocol-v2/EvaluationStratumV2-2.0.schema.json,sha256=JrH5YJANNiM1HU9s-j0st4S5re7h4VNJELontk6ASTI,16913
49
+ equiframe_benchmark/schemas/protocol-v2/FreezeDefinitionV2-2.0.schema.json,sha256=UoxKul4sy98Z66LpVQXh9FmZyCBjeo9lRaiSCoQRB58,137621
50
+ equiframe_benchmark/schemas/protocol-v2/FreezeManifestV2-2.0.schema.json,sha256=c_G5XGmtpEYllsyoGUhuSof0LiXux9-Va1qnSnnPY-s,144291
51
+ equiframe_benchmark/schemas/protocol-v2/PrincipalContrastV2-2.0.schema.json,sha256=bk8AAPfA39J7sYfe5_d781naaXxX3DLGD4u0TiRexiE,13277
52
+ equiframe_benchmark/schemas/protocol-v2/ProtocolReviewV2-2.0.schema.json,sha256=IaOUNobviiPGjHBr0v_nt-3R60b-DQ8K6jYlrCzRS0A,984
53
+ equiframe_benchmark/schemas/protocol-v2/SignedMetricV2-2.0.schema.json,sha256=nDcMR1S9KOGBku72bOqH5QmSUs1sP6bXaYKffXzR2ww,859
54
+ equiframe_benchmark/schemas/protocol-v2/StudyCaseResultV2-2.0.schema.json,sha256=bseEJ6M-kuBUOrRHAauX2yjek-XL-ykt0BrVsbyyyZ8,111449
55
+ equiframe_benchmark/schemas/protocol-v2/StudyResultV2-2.0.schema.json,sha256=9H1mSh-_NfwLgsuUuynPOWdYERyHV3o2pvVCti5EWEw,193449
56
+ equiframe_benchmark/schemas/protocol-v2/TableExportV2-2.0.schema.json,sha256=jAtFz1rhuhmR0PLdczkWQuLOGggnOCjF1aGdQqquKU4,1567
57
+ equiframe_benchmark-0.12.4.dist-info/METADATA,sha256=_HRjr0RPCRp9EVuhyMQRsuIoBh1Zk2azDXschj_CG18,19178
58
+ equiframe_benchmark-0.12.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
59
+ equiframe_benchmark-0.12.4.dist-info/entry_points.txt,sha256=R6ruYCoE117I0xgmTRNl9dNYEN14D-pv22SZyUFpXqM,128
60
+ equiframe_benchmark-0.12.4.dist-info/licenses/LICENSE,sha256=KbNEP-20dJmFziJl6wTjeU3TZvhJ-TBkOKnZJOeuGjA,1070
61
+ equiframe_benchmark-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md,sha256=J33JUToGEm9bDOCfXCR7xBeJXKLKMcofT16hgKMtVbA,376
62
+ equiframe_benchmark-0.12.4.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.28.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ equiframe-benchmark = equiframe_benchmark.cli:main
3
+ equiframe-benchmark-pilot = equiframe_benchmark.pilot:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Hernán Troya
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,8 @@
1
+ # Third-party notices
2
+
3
+ The original EquiFrame Benchmark code and accompanying project documentation,
4
+ examples and tools are licensed under MIT; see LICENSE.
5
+
6
+ Dependencies, including equiframe, are separate distributions and retain their
7
+ own license terms and notices. Downloaded datasets, external R installations
8
+ and the thesis manuscript are not relicensed by this package.