equiframe-benchmark 0.12.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- equiframe_benchmark/PROTOCOL_V2.md +172 -0
- equiframe_benchmark/__init__.py +284 -0
- equiframe_benchmark/__main__.py +3 -0
- equiframe_benchmark/_scale_workload.py +375 -0
- equiframe_benchmark/calibration_types.py +112 -0
- equiframe_benchmark/cli.py +196 -0
- equiframe_benchmark/commands.py +149 -0
- equiframe_benchmark/commands_scale.py +38 -0
- equiframe_benchmark/commands_step10.py +275 -0
- equiframe_benchmark/commands_v2.py +206 -0
- equiframe_benchmark/cost_table.py +62 -0
- equiframe_benchmark/costs.py +788 -0
- equiframe_benchmark/distribution.py +932 -0
- equiframe_benchmark/evaluation.py +614 -0
- equiframe_benchmark/evaluation_v2.py +428 -0
- equiframe_benchmark/fixtures.py +299 -0
- equiframe_benchmark/freeze.py +869 -0
- equiframe_benchmark/freeze_v2.py +412 -0
- equiframe_benchmark/integration.py +655 -0
- equiframe_benchmark/io.py +29 -0
- equiframe_benchmark/models.py +313 -0
- equiframe_benchmark/models_v2.py +13 -0
- equiframe_benchmark/mutations.py +34 -0
- equiframe_benchmark/oracle.py +264 -0
- equiframe_benchmark/pairwise.py +248 -0
- equiframe_benchmark/pilot.py +254 -0
- equiframe_benchmark/py.typed +0 -0
- equiframe_benchmark/reconciliation.py +294 -0
- equiframe_benchmark/reproduction.py +542 -0
- equiframe_benchmark/scale_io.py +19 -0
- equiframe_benchmark/scale_process.py +1077 -0
- equiframe_benchmark/scale_types.py +120 -0
- equiframe_benchmark/scaling.py +529 -0
- equiframe_benchmark/schemas/protocol-v2/CommonSupportV2-2.0.schema.json +327 -0
- equiframe_benchmark/schemas/protocol-v2/ContrastSpecV2-2.0.schema.json +37 -0
- equiframe_benchmark/schemas/protocol-v2/EvaluationConfigV2-2.0.schema.json +184 -0
- equiframe_benchmark/schemas/protocol-v2/EvaluationReportV2-2.0.schema.json +1237 -0
- equiframe_benchmark/schemas/protocol-v2/EvaluationStratumV2-2.0.schema.json +747 -0
- equiframe_benchmark/schemas/protocol-v2/FreezeDefinitionV2-2.0.schema.json +5951 -0
- equiframe_benchmark/schemas/protocol-v2/FreezeManifestV2-2.0.schema.json +6222 -0
- equiframe_benchmark/schemas/protocol-v2/PrincipalContrastV2-2.0.schema.json +578 -0
- equiframe_benchmark/schemas/protocol-v2/ProtocolReviewV2-2.0.schema.json +56 -0
- equiframe_benchmark/schemas/protocol-v2/SignedMetricV2-2.0.schema.json +51 -0
- equiframe_benchmark/schemas/protocol-v2/StudyCaseResultV2-2.0.schema.json +4881 -0
- equiframe_benchmark/schemas/protocol-v2/StudyResultV2-2.0.schema.json +8370 -0
- equiframe_benchmark/schemas/protocol-v2/TableExportV2-2.0.schema.json +75 -0
- equiframe_benchmark/selection.py +178 -0
- equiframe_benchmark/snapshot.py +24 -0
- equiframe_benchmark/splits.py +92 -0
- equiframe_benchmark/study.py +526 -0
- equiframe_benchmark/study_bundle.py +379 -0
- equiframe_benchmark/study_bundle_v2.py +388 -0
- equiframe_benchmark/study_v2.py +544 -0
- equiframe_benchmark/tables.py +347 -0
- equiframe_benchmark/tables_v2.py +340 -0
- equiframe_benchmark/training_types.py +52 -0
- equiframe_benchmark-0.12.4.dist-info/METADATA +343 -0
- equiframe_benchmark-0.12.4.dist-info/RECORD +62 -0
- equiframe_benchmark-0.12.4.dist-info/WHEEL +4 -0
- equiframe_benchmark-0.12.4.dist-info/entry_points.txt +3 -0
- equiframe_benchmark-0.12.4.dist-info/licenses/LICENSE +21 -0
- equiframe_benchmark-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md +8 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Selection audit stays in the benchmark; only vectors cross into the library."""
|
|
2
|
+
|
|
3
|
+
from equiframe import NormalTrainingProfiles
|
|
4
|
+
from pydantic import model_validator
|
|
5
|
+
from typing_extensions import Self
|
|
6
|
+
|
|
7
|
+
from equiframe_benchmark.models import Digest, Document, Text, digest
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TrainingRecord(Document):
|
|
11
|
+
scenario_id: Text
|
|
12
|
+
candidate_id: Text
|
|
13
|
+
group_id: Digest
|
|
14
|
+
candidate_sha256: Digest
|
|
15
|
+
oracle_sha256: Digest
|
|
16
|
+
profile_sha256: Digest
|
|
17
|
+
vector_sha256: Digest
|
|
18
|
+
execution_sha256: Digest
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class TrainingManifest(Document):
|
|
22
|
+
purpose: str = "normal_training"
|
|
23
|
+
catalog_sha256: Digest
|
|
24
|
+
splits_sha256: Digest
|
|
25
|
+
records: tuple[TrainingRecord, ...]
|
|
26
|
+
|
|
27
|
+
@model_validator(mode="after")
|
|
28
|
+
def unique(self) -> Self:
|
|
29
|
+
if self.purpose != "normal_training" or not self.records:
|
|
30
|
+
raise ValueError("nonempty normal training selection required")
|
|
31
|
+
for field in ("candidate_id", "execution_sha256", "profile_sha256"):
|
|
32
|
+
if len({getattr(r, field) for r in self.records}) != len(self.records):
|
|
33
|
+
raise ValueError("duplicate training evidence")
|
|
34
|
+
return self
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class TrainingSelection(Document):
|
|
38
|
+
manifest: TrainingManifest
|
|
39
|
+
training: NormalTrainingProfiles
|
|
40
|
+
|
|
41
|
+
@model_validator(mode="after")
|
|
42
|
+
def consistent(self) -> Self:
|
|
43
|
+
if digest(self.manifest) != self.training.selection_manifest_sha256:
|
|
44
|
+
raise ValueError("selection manifest mismatch")
|
|
45
|
+
if len(self.manifest.records) != len(self.training.profiles):
|
|
46
|
+
raise ValueError("selection size mismatch")
|
|
47
|
+
if any(
|
|
48
|
+
record.vector_sha256 != digest(vector)
|
|
49
|
+
for record, vector in zip(self.manifest.records, self.training.profiles, strict=True)
|
|
50
|
+
):
|
|
51
|
+
raise ValueError("selected vectors do not match the recorded evidence")
|
|
52
|
+
return self
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: equiframe-benchmark
|
|
3
|
+
Version: 0.12.4
|
|
4
|
+
Summary: EquiFrame Verify, paired evaluation and reproducible experimental evidence
|
|
5
|
+
Project-URL: Homepage, https://github.com/htroya/equiframe
|
|
6
|
+
Project-URL: Source, https://github.com/htroya/equiframe
|
|
7
|
+
Project-URL: Documentation, https://github.com/htroya/equiframe/blob/main/docs/index.md
|
|
8
|
+
Project-URL: Issue Tracker, https://github.com/htroya/equiframe/issues
|
|
9
|
+
Author: Hernán Troya
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
License-File: THIRD_PARTY_NOTICES.md
|
|
13
|
+
Keywords: benchmark,data-validation,dataframes,paired-evaluation,reproducibility
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: <3.14,>=3.10
|
|
24
|
+
Requires-Dist: equiframe[arrow]==0.12.4
|
|
25
|
+
Requires-Dist: packaging<27,>=26
|
|
26
|
+
Requires-Dist: tomli<3,>=2.2; python_version < '3.11'
|
|
27
|
+
Requires-Dist: typing-extensions<5,>=4.14
|
|
28
|
+
Provides-Extra: engines
|
|
29
|
+
Requires-Dist: equiframe[engines]==0.12.4; extra == 'engines'
|
|
30
|
+
Provides-Extra: ml
|
|
31
|
+
Requires-Dist: equiframe[ml]==0.12.4; extra == 'ml'
|
|
32
|
+
Provides-Extra: resources
|
|
33
|
+
Requires-Dist: psutil==7.2.2; extra == 'resources'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# EquiFrame Benchmark 0.12.4
|
|
37
|
+
|
|
38
|
+
> Release target: **0.12.4**. Check the official index for availability and
|
|
39
|
+
> verify the identified artifact hashes. Publication requires artifact-bound
|
|
40
|
+
> release checks; prior candidate results keep their own identities.
|
|
41
|
+
|
|
42
|
+
**Experimental evaluation, reproducible evidence and bounded engineering costs.**
|
|
43
|
+
|
|
44
|
+
[Principal paired protocol V2](PROTOCOL_V2.md) provides a separately versioned
|
|
45
|
+
freeze → native study → paired evaluation → seven tables → portable reproduction
|
|
46
|
+
route for the three primary families and three engine pairs. V1 readers,
|
|
47
|
+
fingerprints and common-support results remain unchanged. Earlier protocol
|
|
48
|
+
work included twenty-four focused engineering checks, with API/CLI roundtrips
|
|
49
|
+
over fictional held-out native cases and portable reproduction. Those results
|
|
50
|
+
retain their original source and artifact identities; they do not establish
|
|
51
|
+
0.12.4 validation or scientific approval. Final protocol review remains required.
|
|
52
|
+
|
|
53
|
+
`equiframe-benchmark` is separate from the core `equiframe` package. It provides
|
|
54
|
+
scenario catalogs, independent-review records, grouped splits, reviewed normal
|
|
55
|
+
selection, frozen studies, paired evaluation, reproducible tables/bundles and
|
|
56
|
+
resource measurements. The core never imports it or receives oracle labels as
|
|
57
|
+
detector features.
|
|
58
|
+
|
|
59
|
+
## Release scope and compatibility
|
|
60
|
+
|
|
61
|
+
This distribution is paired exactly with `equiframe[arrow]==0.12.4` and
|
|
62
|
+
supports Python `>=3.10,<3.14`. Original code is licensed under [MIT](LICENSE);
|
|
63
|
+
dependencies retain their own terms in [third-party notices](THIRD_PARTY_NOTICES.md).
|
|
64
|
+
|
|
65
|
+
The owner authorized publication on 2026-10-27 and confirmed local manual
|
|
66
|
+
publication on 2026-10-28. Uploads use an ephemeral local API token, revoked
|
|
67
|
+
after use; no GitHub Actions job or stored GitHub publishing secret is used.
|
|
68
|
+
The release checklist binds the inspected commit and all four artifact hashes,
|
|
69
|
+
checks the security review and clean consumers, and promotes the same bytes.
|
|
70
|
+
A version label or this README does not establish a successful upload.
|
|
71
|
+
|
|
72
|
+
The benchmark's scientific protocols and APIs remain experimental. Software
|
|
73
|
+
correctness tests preserve their documented contracts but do not establish
|
|
74
|
+
real-data accuracy, reviewed corpus labels, independent human acceptance,
|
|
75
|
+
external adoption or a deployed service/data warehouse.
|
|
76
|
+
|
|
77
|
+
Hernán Troya (`htroya`) is the confirmed maintenance, release and security owner,
|
|
78
|
+
with no response-time SLA. Documentation and reporting routes share the
|
|
79
|
+
[EquiFrame repository](https://github.com/htroya/equiframe).
|
|
80
|
+
Sensitive reports use its
|
|
81
|
+
[private reporting channel](https://github.com/htroya/equiframe/security/advisories/new).
|
|
82
|
+
For each release, the maintainer must enable public issues and private reporting
|
|
83
|
+
and verify their availability.
|
|
84
|
+
Historical Alpha artifacts and results retain their original identity and scope.
|
|
85
|
+
|
|
86
|
+
The core wheel includes the self-contained English handbook with API, semantics,
|
|
87
|
+
compatibility, contribution, security, dependency/SBOM and release procedures:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from importlib.resources import files
|
|
91
|
+
|
|
92
|
+
print(files("equiframe").joinpath("docs/index.md").read_text(encoding="utf-8"))
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
From an authorized monorepo checkout the same guide is at
|
|
96
|
+
[core handbook](https://github.com/htroya/equiframe/blob/main/docs/index.md); installed consumers do not need
|
|
97
|
+
repository access. This README is included in distribution metadata. Benchmark
|
|
98
|
+
sdists also include examples, tests, tools and JSON schemas.
|
|
99
|
+
|
|
100
|
+
## Install exact artifacts, then run outside the checkout
|
|
101
|
+
|
|
102
|
+
Obtain both identified wheels and a reviewed dependency wheelhouse for your
|
|
103
|
+
interpreter/platform/profile. Verify supplied SHA-256 values before installation.
|
|
104
|
+
In a fresh directory outside the repository, using a POSIX shell:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
python3.12 -m venv .venv
|
|
108
|
+
. .venv/bin/activate
|
|
109
|
+
CORE=/absolute/path/equiframe-0.12.4-py3-none-any.whl
|
|
110
|
+
BENCH=/absolute/path/equiframe_benchmark-0.12.4-py3-none-any.whl
|
|
111
|
+
WHEELHOUSE=/absolute/path/reviewed-wheelhouse
|
|
112
|
+
python -m pip install --no-index --find-links "$WHEELHOUSE" "${CORE}[engines]" "${BENCH}[resources]"
|
|
113
|
+
python -m pip check
|
|
114
|
+
python -I -c "import equiframe, equiframe_benchmark; print(equiframe.__file__); print(equiframe_benchmark.__file__)"
|
|
115
|
+
python -I -m equiframe_benchmark.pilot --output pilot-synthetic
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Absolute paths are placeholders for verified local files. Missing compatible
|
|
119
|
+
binary dependencies are a blocker; do not drop hashes or switch to an unrelated
|
|
120
|
+
index package. Avoid editable installs/PYTHONPATH as artifact validation. Windows
|
|
121
|
+
can use `py -3.12 -m venv .venv` and `.venv\Scripts\python.exe` directly without
|
|
122
|
+
changing activation security policy. Command conventions are not platform tests.
|
|
123
|
+
|
|
124
|
+
The benchmark base requires Arrow through core and `packaging>=26,<27`.
|
|
125
|
+
`engines` adds pandas/Polars/DuckDB; `ml` adds the core ML extra;
|
|
126
|
+
`resources` adds `psutil==7.2.2`. Choose reviewed dependency pins for the required
|
|
127
|
+
profiles. Metadata admission does not prove all platforms or range combinations.
|
|
128
|
+
Historical measured cells are Linux x86_64/Python 3.12.14 and 3.13.5; a current
|
|
129
|
+
candidate needs its own exact-artifact receipt. Manual CI configuration is not a
|
|
130
|
+
remote job result, and Windows/macOS/ARM64 are not newly claimed as verified.
|
|
131
|
+
|
|
132
|
+
The pilot destination must be new. `equiframe-benchmark-pilot --output DIR` is an
|
|
133
|
+
installed equivalent entry point. The synthetic pilot produces 5 scenarios,
|
|
134
|
+
35 candidates, 15 profiles and 4 dependency groups. **All 35 reviews remain
|
|
135
|
+
pending**. Its 15 mutations are artifact injections; its 15 control
|
|
136
|
+
transformations execute natively. These are not a completed thesis sample or
|
|
137
|
+
trained scientific model.
|
|
138
|
+
|
|
139
|
+
## Review, grouping and normal selection
|
|
140
|
+
|
|
141
|
+
Public models include `Scenario`, `Catalog`, `Candidate`, `ReviewPacket`,
|
|
142
|
+
`ReviewAssignment`, `Review`, `OracleResult`, `SplitManifest`, `TrainingSelection`
|
|
143
|
+
and `CalibrationSelection`. `load_document`, `save_document` and `digest` use
|
|
144
|
+
strict canonical JSON and explicit evidence identity.
|
|
145
|
+
|
|
146
|
+
This snippet continues from the synthetic pilot above:
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
from pathlib import Path
|
|
150
|
+
import equiframe_benchmark as eb
|
|
151
|
+
|
|
152
|
+
root = Path("pilot-synthetic")
|
|
153
|
+
catalog = eb.load_document(eb.Catalog, root / "catalog.json")
|
|
154
|
+
splits = eb.load_document(eb.SplitManifest, root / "splits.json")
|
|
155
|
+
eb.validate_splits(catalog, splits)
|
|
156
|
+
print(eb.split_counts(catalog, splits))
|
|
157
|
+
scenario = catalog.scenarios[0]
|
|
158
|
+
candidate = eb.load_document(eb.Candidate, root / "nulls_types" / "pandas-candidate.json")
|
|
159
|
+
packet = eb.review_packet(scenario, candidate, directory=Path("review-packet"))
|
|
160
|
+
pending = eb.evaluate_oracle(scenario, candidate, packet=packet, work_dir=Path("oracle-evidence"))
|
|
161
|
+
assert pending.status == "pending" and pending.label is None
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Only the blinded packet goes to the reviewer. Keep catalog/candidate metadata,
|
|
165
|
+
alerts and other reviewers' decisions separate. Record two independent reviewers
|
|
166
|
+
and an arbiter in an assignment before review, with UTC timestamps. Reviews
|
|
167
|
+
require packet hash, identity declaration, decision, rationale, timestamp and
|
|
168
|
+
blind attestation. Resolution is permitted for disagreement by the assigned
|
|
169
|
+
arbiter. Do not generate human approvals from comparator output. Hashes and JSON
|
|
170
|
+
names do not authenticate a human or establish true independence.
|
|
171
|
+
|
|
172
|
+
`dependency_groups` closes transitive declared/artifact dependencies before
|
|
173
|
+
`make_splits`; `validate_splits` and `split_counts` retain group/partition scope.
|
|
174
|
+
`select_normal_training` and `select_normal_calibration` require candidate,
|
|
175
|
+
oracle, execution and profile maps with exact linked identities, one family and
|
|
176
|
+
feature definition, confirmed normal O, and the appropriate disjoint partition.
|
|
177
|
+
The pending pilot is not eligible normal training evidence.
|
|
178
|
+
|
|
179
|
+
`audit_detector_selections` checks shared catalog/split, compatible features,
|
|
180
|
+
independent groups and no reused scenarios/executions/profiles. Respect
|
|
181
|
+
`eligible`/`not_estimable`, preserve counts and justify minimum support before
|
|
182
|
+
freeze. Core `fit_detectors`, `score_detectors`, `calibrate_detectors`, `detect`
|
|
183
|
+
and `detect_component` separate fitting, scoring, calibration and inference.
|
|
184
|
+
No anomaly/no-alert state is a comparison oracle; test never fits the model.
|
|
185
|
+
|
|
186
|
+
## Public workflow reference
|
|
187
|
+
|
|
188
|
+
The exhaustive experimental export namespace is `equiframe_benchmark.__all__`.
|
|
189
|
+
Use `inspect.signature(eb.function)` and each model's `model_json_schema()` for
|
|
190
|
+
installed signatures/defaults. These are distinct families:
|
|
191
|
+
|
|
192
|
+
| Family | Entry points |
|
|
193
|
+
|---|---|
|
|
194
|
+
| Blinded review | `review_packet`, `evaluate_oracle`, `oracle_verification_bridge` |
|
|
195
|
+
| Dependency splits | `dependency_groups`, `make_splits`, `validate_splits`, `split_counts` |
|
|
196
|
+
| Reviewed selections | `select_normal_training`, `select_normal_calibration`, `audit_detector_selections` |
|
|
197
|
+
| Snapshot consumer | Compatibility aliases of core `SnapshotClient`, `SnapshotDataSource`, `snapshot_fingerprint` |
|
|
198
|
+
| Three-engine integration | `run_integration`, `read_integration` |
|
|
199
|
+
| Domain-configurable reconciliation | `reconcile_awards`, `AwardReconciliationSpec` |
|
|
200
|
+
| Freeze/study | `audit_freeze`, `create_freeze`, `read_freeze`, `run_study`, `read_study` |
|
|
201
|
+
| Evaluation | `evaluate_records`, `evaluate_pairwise`, `evaluation_protocol_sha256` |
|
|
202
|
+
| Tables | `table_payloads`, `write_tables`, `read_tables` |
|
|
203
|
+
| Scientific cost protocol | `measure_costs`, `read_costs`, `write_cost_table` |
|
|
204
|
+
| Record reproduction | `make_reproduction_record`, `compare_reproductions` |
|
|
205
|
+
| Generic bundles | `create_bundle`, `read_bundle`, `bundle_artifact`, `reproduce_bundle` |
|
|
206
|
+
| Study bundles | `bundle_study`, `regenerate_study_record`, `reproduce_study_bundle` |
|
|
207
|
+
| Engineering scale | `measure_scale`, `read_scale`, `summarize_scale` |
|
|
208
|
+
| Distribution inventory | `inspect_distribution`, `DistributionReadiness`, `NameCheck` |
|
|
209
|
+
|
|
210
|
+
Snapshot consumers pin a declared publication and verify artifacts from an
|
|
211
|
+
explicit trusted client. `run_integration` executes each of three registered
|
|
212
|
+
native implementations once, then compares three pairs. A separate optional
|
|
213
|
+
expected output is fixed before execution. Shared captures do not become
|
|
214
|
+
additional independent observations when reused in several pair comparisons.
|
|
215
|
+
If a native execution fails, its expected-output comparison is recorded as
|
|
216
|
+
`execution_unavailable`; the operation stays incomplete and healthy independent
|
|
217
|
+
pairs retain their evidence. The synthetic date fixture uses Pandas' native UTC
|
|
218
|
+
microsecond dtype before extracting date32, including empty and NULL values,
|
|
219
|
+
so it does not require Arrow's separate C++ timezone database on Windows.
|
|
220
|
+
Client/DW authentication, service writer coordination and production acceptance
|
|
221
|
+
are not implemented by the consumer interface.
|
|
222
|
+
|
|
223
|
+
## Freeze, study and reproduction
|
|
224
|
+
|
|
225
|
+
`FreezeDefinition` records catalog, grouped splits, cases, family model slots,
|
|
226
|
+
reviewed selections, prior decisions, files/hashes, environment and evaluation/cost
|
|
227
|
+
protocols. Pilot freeze verifies technical readiness while retaining scientific
|
|
228
|
+
blockers. Final freeze requires those blockers resolved; it does not manufacture
|
|
229
|
+
reviews or fit on test data.
|
|
230
|
+
|
|
231
|
+
The following are command **templates** using your own reviewed files and trusted
|
|
232
|
+
registry factory; they are not a complete runnable scientific experiment:
|
|
233
|
+
|
|
234
|
+
```text
|
|
235
|
+
equiframe-benchmark freeze definition.json --output freeze.json
|
|
236
|
+
equiframe-benchmark verify-freeze freeze.json
|
|
237
|
+
equiframe-benchmark run-study freeze.json --registry my_etl:create_registry --output studies
|
|
238
|
+
equiframe-benchmark verify-study studies/study_ID
|
|
239
|
+
equiframe-benchmark evaluate studies/study_ID --output evaluation.json
|
|
240
|
+
equiframe-benchmark tables studies/study_ID --output regenerated-tables
|
|
241
|
+
equiframe-benchmark bundle-study studies/study_ID --output portable-study
|
|
242
|
+
equiframe-benchmark reproduce-study portable-study --expected-sha256 TRUSTED_BUNDLE_MANIFEST_SHA256
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
Factory references import and execute trusted code explicitly. JSON requests do
|
|
246
|
+
not authorize arbitrary plugins or contain credentials. The frozen historical
|
|
247
|
+
main comparison uses confirmed O and common complete R/M/B/H support, retaining
|
|
248
|
+
individual support, partial alerts, failures and exclusions. `evaluate_pairwise`
|
|
249
|
+
adds six descriptive pairwise supports with their own denominators; it preserves
|
|
250
|
+
the original report/protocol/six CSV and remains `scientific=False`. Using that
|
|
251
|
+
analysis as a new scientific primary protocol requires a new freeze.
|
|
252
|
+
|
|
253
|
+
Undefined precision, recall, F1 and FPR remain absent with reasons. Average
|
|
254
|
+
precision requires two classes and a common family/model. Bootstrap samples
|
|
255
|
+
whole groups and requires a prior independence rationale. CSVs regenerate from
|
|
256
|
+
verified records; editing a CSV and recomputing its hash does not create valid
|
|
257
|
+
changed evidence.
|
|
258
|
+
|
|
259
|
+
Study bundles preserve original captures/manifests and regenerate configuration,
|
|
260
|
+
splits, labels, decisions and tables without rerunning ETL. They do not certify a
|
|
261
|
+
colleague independently repeated the experiment. Raw costs are retained without
|
|
262
|
+
requiring identical timings on reproduction. The distribution inventory does not
|
|
263
|
+
reserve a name, resolve licensing, install a package or authorize publication.
|
|
264
|
+
|
|
265
|
+
## Cost protocols and engineering scale
|
|
266
|
+
|
|
267
|
+
`measure_costs` uses a frozen scientific `CostProtocol`: isolated profile+M versus
|
|
268
|
+
full comparison over already captured outputs, one warmup and five alternating
|
|
269
|
+
repetitions per method. Initial ETL/capture is excluded. `cost-table` retains all
|
|
270
|
+
twelve raw observations, warmups, failures and reasons. Repetitions do not increase
|
|
271
|
+
the independent detection sample. Missing monitoring is never imputed as zero.
|
|
272
|
+
|
|
273
|
+
The separate engineering protocol `ScaleSpec`/`ScaleTrial`/`ScaleRun` measures
|
|
274
|
+
bounded synthetic integer/text/NULL workloads. See the installed core handbook's
|
|
275
|
+
`docs/performance.md` for a complete small script, stage boundaries and budgets.
|
|
276
|
+
Source distributions also contain `examples/scale_local.py`, which defaults to
|
|
277
|
+
1k rows; 100k/1M require explicit `--rows` and larger recorded budgets.
|
|
278
|
+
|
|
279
|
+
One isolated trusted worker records fresh/warm process groups with uncontrolled
|
|
280
|
+
OS caches, nine stages, sampled RSS/CPU/I/O, disk and evidence bytes. RSS/disk
|
|
281
|
+
thresholds are reactive and can overshoot; they are not hard quotas. A valid
|
|
282
|
+
completion marker, expected verdict and verified inventory are required.
|
|
283
|
+
`complete` and `resource_measurement_complete` are separate: a successful
|
|
284
|
+
operation can have incomplete observations.
|
|
285
|
+
|
|
286
|
+
Restricted native pandas/DuckDB baselines provide less evidence; time ratios
|
|
287
|
+
are not equivalent-service speed rankings. DataComPy is unpinned/unavailable,
|
|
288
|
+
with no invented timing. Kill, missing finalization, failed/partial measurement,
|
|
289
|
+
ENOSPC and last-batch differences stay visible. Receipts are location-bound;
|
|
290
|
+
reproduction uses the specification and exact wheels, not an implied portable run.
|
|
291
|
+
|
|
292
|
+
## CLI exits and safe interpretation
|
|
293
|
+
|
|
294
|
+
`equiframe-benchmark --help` lists the installed commands, including catalog
|
|
295
|
+
`validate`, `split`, `pilot`, integration, freeze/study, evaluation, costs and scale.
|
|
296
|
+
|
|
297
|
+
- Study/integration operations distinguish complete without alert (0), complete
|
|
298
|
+
with alert/difference (1), invalid input/freeze (2) and partial/unavailable (3)
|
|
299
|
+
- Integrity `verify-*` commands can return 0 for intact evidence of a historical
|
|
300
|
+
incomplete run; read the stored semantic state. They do not turn it into success
|
|
301
|
+
- Reproduction disagreement uses 1; invalid evidence uses a failure code
|
|
302
|
+
- `costs` uses 3 for incomplete measurement
|
|
303
|
+
- `scale` and `verify-scale` use 3 for unsuccessful/incomplete measurement or
|
|
304
|
+
invalid finalization. Their zero is expected-workload completion, not equivalence
|
|
305
|
+
|
|
306
|
+
The historical core `equiframe.benchmark` placeholder is quarantined and not a
|
|
307
|
+
compatible shortcut into this package. Do not reenable its removed metrics or
|
|
308
|
+
infer scientific results from synthetic demonstrations.
|
|
309
|
+
|
|
310
|
+
## Development and contribution
|
|
311
|
+
|
|
312
|
+
Operational errors retain their original first stderr line and exit status,
|
|
313
|
+
then add one fixed-text `hint[category]: ...` line. For example, an existing
|
|
314
|
+
output recommends a new path instead of overwriting evidence; a missing file
|
|
315
|
+
recommends checking the input and parent directory. Hints omit raw exception
|
|
316
|
+
messages, data and paths. Automation should read the exit status and first
|
|
317
|
+
line, not compare the complete stderr text. `scale` retains exit 3 for its
|
|
318
|
+
failed operation, including invalid specifications.
|
|
319
|
+
The standalone `equiframe-benchmark-pilot` entrypoint also uses fixed hints;
|
|
320
|
+
its operation failures retain exit 1 and print `pilot_operation_failed`.
|
|
321
|
+
|
|
322
|
+
From `packages/equiframe-benchmark`, with the matching core wheel already built:
|
|
323
|
+
|
|
324
|
+
```bash
|
|
325
|
+
uv sync --locked --all-extras --reinstall-package equiframe
|
|
326
|
+
uv run --locked --all-extras ruff check .
|
|
327
|
+
uv run --locked --all-extras ruff format --check .
|
|
328
|
+
uv run --locked --all-extras mypy
|
|
329
|
+
uv run --locked --all-extras python tools/generate_assets.py --check
|
|
330
|
+
uv run --locked --all-extras pytest -q
|
|
331
|
+
uv run --locked --all-extras python -m build --no-isolation
|
|
332
|
+
uv run --locked --all-extras twine check --strict dist/*.whl dist/*.tar.gz
|
|
333
|
+
uv run --locked --all-extras python tools/verify_distribution.py
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
`--reinstall-package equiframe` prevents stale local dependency caching while a
|
|
337
|
+
package version is unchanged. From this directory, the byte-bound candidate driver is
|
|
338
|
+
`python ../../tools/check_candidate.py --package . --output /absolute/path/new-checks.json`.
|
|
339
|
+
These are checks to execute, not claims they already passed this artifact.
|
|
340
|
+
Source tests, exact installed tests and remote CI are distinct evidence. Preserve
|
|
341
|
+
historical fixtures, safe failures and methodological boundaries. Follow the core
|
|
342
|
+
handbook's contribution, upgrade, security and release procedures. No public
|
|
343
|
+
publication or production deployment is authorized by building these artifacts.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
equiframe_benchmark/__init__.py,sha256=CV76hypwD3EqlNOt31h3H4hy4fWZEG3G4DHad_251YI,7253
|
|
2
|
+
equiframe_benchmark/__main__.py,sha256=pleKgOthX0VewbdCKlEd9TkzE0aImbhDAp8zvJbEroc,49
|
|
3
|
+
equiframe_benchmark/_scale_workload.py,sha256=Yu8ihfuqmh1Qc_VlgGBRlg5P6NW9Jkkji5Ka-VE1970,13731
|
|
4
|
+
equiframe_benchmark/calibration_types.py,sha256=r-sfBQrX9ZuhKOSCdGz2jIKYDekjX8Y7sh8czueg8rA,4534
|
|
5
|
+
equiframe_benchmark/cli.py,sha256=q8GBPyF6LO2uX6cFWMFgemDa5wkyypxn2oqLJPRw9x4,6913
|
|
6
|
+
equiframe_benchmark/commands.py,sha256=qoLAnIwV3u1RrAxfPCwFnyKkbDrMu7IIXcO3vuTEFs8,5619
|
|
7
|
+
equiframe_benchmark/commands_scale.py,sha256=6B8OhI7oAYr2EyzPRsCGmIbV_Q5IB4JXVkl0KXUuKKM,1526
|
|
8
|
+
equiframe_benchmark/commands_step10.py,sha256=-j8moHHnMTPXW-9cJLEPvjmEJzqVDjzglxgvyYS5WIQ,10309
|
|
9
|
+
equiframe_benchmark/commands_v2.py,sha256=ID6Za42JrzdCEYQEs8F6_LWvvievOVcNeCU-L5tqLt8,7917
|
|
10
|
+
equiframe_benchmark/cost_table.py,sha256=oVIBDKq-RKPm3kb8OrhPS6JPqldXopzrhtMRhccB7SY,2025
|
|
11
|
+
equiframe_benchmark/costs.py,sha256=575dRSSfEiS7hkI3RJvOQTw6Y4daIUU1REPBdPkooyU,32590
|
|
12
|
+
equiframe_benchmark/distribution.py,sha256=_NqIyo_zHQe7xUWTH3GZYGQf3rEn5ZSeb_WkJrF0TiA,39689
|
|
13
|
+
equiframe_benchmark/evaluation.py,sha256=4NXMM-wpDRW9gz4_6A8fGzII_TNzguRLrPilRmfQloo,24475
|
|
14
|
+
equiframe_benchmark/evaluation_v2.py,sha256=EshYcCXDv7ld5Aqv-0BtaK0LgZIpdpIkcStEepGZaWM,16083
|
|
15
|
+
equiframe_benchmark/fixtures.py,sha256=RZOhvPUW6gZK5IypHRJdQIdYg8orK5o5dgkZLTgF2UE,11306
|
|
16
|
+
equiframe_benchmark/freeze.py,sha256=_O80pzC8-FuKFSExF0UHAIkWz04TkefiMNcqmoE8E8w,37311
|
|
17
|
+
equiframe_benchmark/freeze_v2.py,sha256=N2r0CFPVSS1BuWyjU6JWoUdVB2S6gMY-dXLb4c30dak,17984
|
|
18
|
+
equiframe_benchmark/integration.py,sha256=xovO12X193SuKFt5vfnQDxx_a-AVaI1DH9XV2NiRiW0,28811
|
|
19
|
+
equiframe_benchmark/io.py,sha256=M53n9ZJwCxhgw58nEunGQAA-lqoG9HBkV3knRU56UMo,845
|
|
20
|
+
equiframe_benchmark/models.py,sha256=SPDcHfIImoX4V0amNvD7QzfY5PU38PFqn8uZJZlPeWY,12262
|
|
21
|
+
equiframe_benchmark/models_v2.py,sha256=vJNdYhdWS_RAw-IPBTDwHp1_eeiwk3feV0lZD1KuXZs,386
|
|
22
|
+
equiframe_benchmark/mutations.py,sha256=W6EhzTq7KqCn4iQUoD0Vc3YEYDjmxruQl6oGbvSrSz8,1195
|
|
23
|
+
equiframe_benchmark/oracle.py,sha256=opDtIa9YKYY1SONdotnh-BuD6Smm7eTelzanMyCoYzY,10495
|
|
24
|
+
equiframe_benchmark/pairwise.py,sha256=kIN6ooxdWezpGu02FSSaoGAaylvF3Pnb--31O5-_lFw,9606
|
|
25
|
+
equiframe_benchmark/pilot.py,sha256=emTNlfUOh0Q77lZchpwG4ufGRmlTfdy94DUNdwAc3vs,10238
|
|
26
|
+
equiframe_benchmark/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
27
|
+
equiframe_benchmark/reconciliation.py,sha256=3KlXk1PMbzOev9yZTnlGplqTvOQ5I0WIxHkZrxGXmc4,12452
|
|
28
|
+
equiframe_benchmark/reproduction.py,sha256=anpP5nVZpj_OLPFMrsa0fgI0kY9zKNOLw_0qEUtyLtY,23477
|
|
29
|
+
equiframe_benchmark/scale_io.py,sha256=Bn5rzwzvTpMp7y1srLBVNQ0mgRNk71v76Zpd_UkujmQ,719
|
|
30
|
+
equiframe_benchmark/scale_process.py,sha256=IBn1xndN1kfeMExAe8k156QiR8TTuhjQPGogoK-aHAM,45156
|
|
31
|
+
equiframe_benchmark/scale_types.py,sha256=FxCeSysFVjYHf34XCr2b3x-996YX-OcBjkpcd3B-od4,5076
|
|
32
|
+
equiframe_benchmark/scaling.py,sha256=a8x5ArKft7y2j11qYD6r8mSDgxFiQm8bFj7YrSvCTOk,22246
|
|
33
|
+
equiframe_benchmark/selection.py,sha256=ftpQIoZoi-rCQp4YxuPw0GHi7sByeCeyBWtpC2eLtcA,7819
|
|
34
|
+
equiframe_benchmark/snapshot.py,sha256=ysbnD3qoOaIXZt4ZgOacUuEUDibFvHOQHerCim5DOsQ,689
|
|
35
|
+
equiframe_benchmark/splits.py,sha256=a0mBDF_45jeFSr1xFyxGwz7jTLcBsbM5kEN3ydNvTDg,4009
|
|
36
|
+
equiframe_benchmark/study.py,sha256=oSSrJXJR5Y49-L38yQzcrfn2se0YhNp9kkrDVNOPU7E,22154
|
|
37
|
+
equiframe_benchmark/study_bundle.py,sha256=bscyovidROBcC5spnLuBCRuFyLeBybDhjiala_HIO8Y,16741
|
|
38
|
+
equiframe_benchmark/study_bundle_v2.py,sha256=xw_hH052cjMJlnPXx0c8ZhB_Vf1-Q33YzaYnYYbWHfQ,17207
|
|
39
|
+
equiframe_benchmark/study_v2.py,sha256=GM78b2yo1e9zOZPP4HOBY_O9YvFuMma68evJWPdOYL0,22825
|
|
40
|
+
equiframe_benchmark/tables.py,sha256=ELggQg_9_a60lFx_5Bvezp_UdnDQzC5I9CDrmRbL2RM,12287
|
|
41
|
+
equiframe_benchmark/tables_v2.py,sha256=kA27TGX9wd3ZYKvr5d9oxd8ZadF02lQFe1D5OKWmmmg,12158
|
|
42
|
+
equiframe_benchmark/training_types.py,sha256=xcSYoszPl9XLYR-t_1rjpcyAMekdMoWMUriH7C6YLX0,1846
|
|
43
|
+
equiframe_benchmark/PROTOCOL_V2.md,sha256=pzIzYh17wp_5YIVh_6AFHFrelr6uowGpHRWUSRRRZ-4,8787
|
|
44
|
+
equiframe_benchmark/schemas/protocol-v2/CommonSupportV2-2.0.schema.json,sha256=JxsDFunP9_trEHjVYw3mmr2CMxAYOSfFls3C_wODcNY,7084
|
|
45
|
+
equiframe_benchmark/schemas/protocol-v2/ContrastSpecV2-2.0.schema.json,sha256=sovqKGnKDMUugNXPmshZ90JrIsm0uLMwYP-HHgQXSBM,573
|
|
46
|
+
equiframe_benchmark/schemas/protocol-v2/EvaluationConfigV2-2.0.schema.json,sha256=aNDW-LJT8Rg7lfQjuPdJQ-T0UgpZucXgzbCiMstueAI,3907
|
|
47
|
+
equiframe_benchmark/schemas/protocol-v2/EvaluationReportV2-2.0.schema.json,sha256=pXYsEmmQ89PgaKVRjdvI5VhnMzMt0MkCGhwErqy9z-w,28412
|
|
48
|
+
equiframe_benchmark/schemas/protocol-v2/EvaluationStratumV2-2.0.schema.json,sha256=JrH5YJANNiM1HU9s-j0st4S5re7h4VNJELontk6ASTI,16913
|
|
49
|
+
equiframe_benchmark/schemas/protocol-v2/FreezeDefinitionV2-2.0.schema.json,sha256=UoxKul4sy98Z66LpVQXh9FmZyCBjeo9lRaiSCoQRB58,137621
|
|
50
|
+
equiframe_benchmark/schemas/protocol-v2/FreezeManifestV2-2.0.schema.json,sha256=c_G5XGmtpEYllsyoGUhuSof0LiXux9-Va1qnSnnPY-s,144291
|
|
51
|
+
equiframe_benchmark/schemas/protocol-v2/PrincipalContrastV2-2.0.schema.json,sha256=bk8AAPfA39J7sYfe5_d781naaXxX3DLGD4u0TiRexiE,13277
|
|
52
|
+
equiframe_benchmark/schemas/protocol-v2/ProtocolReviewV2-2.0.schema.json,sha256=IaOUNobviiPGjHBr0v_nt-3R60b-DQ8K6jYlrCzRS0A,984
|
|
53
|
+
equiframe_benchmark/schemas/protocol-v2/SignedMetricV2-2.0.schema.json,sha256=nDcMR1S9KOGBku72bOqH5QmSUs1sP6bXaYKffXzR2ww,859
|
|
54
|
+
equiframe_benchmark/schemas/protocol-v2/StudyCaseResultV2-2.0.schema.json,sha256=bseEJ6M-kuBUOrRHAauX2yjek-XL-ykt0BrVsbyyyZ8,111449
|
|
55
|
+
equiframe_benchmark/schemas/protocol-v2/StudyResultV2-2.0.schema.json,sha256=9H1mSh-_NfwLgsuUuynPOWdYERyHV3o2pvVCti5EWEw,193449
|
|
56
|
+
equiframe_benchmark/schemas/protocol-v2/TableExportV2-2.0.schema.json,sha256=jAtFz1rhuhmR0PLdczkWQuLOGggnOCjF1aGdQqquKU4,1567
|
|
57
|
+
equiframe_benchmark-0.12.4.dist-info/METADATA,sha256=_HRjr0RPCRp9EVuhyMQRsuIoBh1Zk2azDXschj_CG18,19178
|
|
58
|
+
equiframe_benchmark-0.12.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
59
|
+
equiframe_benchmark-0.12.4.dist-info/entry_points.txt,sha256=R6ruYCoE117I0xgmTRNl9dNYEN14D-pv22SZyUFpXqM,128
|
|
60
|
+
equiframe_benchmark-0.12.4.dist-info/licenses/LICENSE,sha256=KbNEP-20dJmFziJl6wTjeU3TZvhJ-TBkOKnZJOeuGjA,1070
|
|
61
|
+
equiframe_benchmark-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md,sha256=J33JUToGEm9bDOCfXCR7xBeJXKLKMcofT16hgKMtVbA,376
|
|
62
|
+
equiframe_benchmark-0.12.4.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hernán Troya
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Third-party notices
|
|
2
|
+
|
|
3
|
+
The original EquiFrame Benchmark code and accompanying project documentation,
|
|
4
|
+
examples and tools are licensed under MIT; see LICENSE.
|
|
5
|
+
|
|
6
|
+
Dependencies, including equiframe, are separate distributions and retain their
|
|
7
|
+
own license terms and notices. Downloaded datasets, external R installations
|
|
8
|
+
and the thesis manuscript are not relicensed by this package.
|