equiframe 0.12.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- equiframe/__init__.py +832 -0
- equiframe/__main__.py +3 -0
- equiframe/_base.py +84 -0
- equiframe/_cli_support.py +125 -0
- equiframe/_comparison_io.py +161 -0
- equiframe/_dataset_provenance.py +146 -0
- equiframe/_decimal.py +22 -0
- equiframe/_diff_capture.py +433 -0
- equiframe/_doctor.py +123 -0
- equiframe/_document_io.py +61 -0
- equiframe/_documents.py +79 -0
- equiframe/_duckdb_transport.py +149 -0
- equiframe/_error_reporting.py +52 -0
- equiframe/_first_use.py +102 -0
- equiframe/_path_compat.py +26 -0
- equiframe/_physical.py +133 -0
- equiframe/_r_review.R +308 -0
- equiframe/_regular_io.py +167 -0
- equiframe/_report_format_v1.py +298 -0
- equiframe/_report_io.py +591 -0
- equiframe/_stability_inventory.py +1421 -0
- equiframe/_timezones.py +257 -0
- equiframe/_vendor/cyclonedx/LICENSE +201 -0
- equiframe/_vendor/cyclonedx/PROVENANCE.md +18 -0
- equiframe/_vendor/cyclonedx/bom-1.6.schema.json +5673 -0
- equiframe/_vendor/cyclonedx/jsf-0.82.schema.json +240 -0
- equiframe/_vendor/cyclonedx/spdx.schema.json +712 -0
- equiframe/adapters.py +454 -0
- equiframe/assertion_types.py +122 -0
- equiframe/assertions.py +271 -0
- equiframe/benchmark.py +101 -0
- equiframe/bundle.py +240 -0
- equiframe/cli.py +693 -0
- equiframe/comparison.py +937 -0
- equiframe/comparison_report.py +660 -0
- equiframe/comparison_types.py +257 -0
- equiframe/compatibility.py +546 -0
- equiframe/contracts.py +879 -0
- equiframe/datasets.py +1301 -0
- equiframe/decision.py +34 -0
- equiframe/destinations.py +295 -0
- equiframe/detector_types.py +431 -0
- equiframe/detectors.py +389 -0
- equiframe/diff.py +192 -0
- equiframe/diff_contracts.py +523 -0
- equiframe/docs/adoption.md +329 -0
- equiframe/docs/api.md +534 -0
- equiframe/docs/arquitecto.md +143 -0
- equiframe/docs/choosing-api.md +84 -0
- equiframe/docs/cientifico.md +180 -0
- equiframe/docs/compatibility.md +254 -0
- equiframe/docs/contributing.md +171 -0
- equiframe/docs/dependencies.md +237 -0
- equiframe/docs/eda.md +166 -0
- equiframe/docs/extending.md +158 -0
- equiframe/docs/fast-ci.md +52 -0
- equiframe/docs/first-use.md +78 -0
- equiframe/docs/index.md +142 -0
- equiframe/docs/ingestion.md +145 -0
- equiframe/docs/inicio.md +129 -0
- equiframe/docs/installed-private.md +97 -0
- equiframe/docs/journeys.md +362 -0
- equiframe/docs/lectura-conversion.md +211 -0
- equiframe/docs/licensing.md +17 -0
- equiframe/docs/maintenance.md +233 -0
- equiframe/docs/migration-ir.md +333 -0
- equiframe/docs/option-audit.md +45 -0
- equiframe/docs/outcome-gates.md +134 -0
- equiframe/docs/performance.md +263 -0
- equiframe/docs/piloto-simulado.md +56 -0
- equiframe/docs/quickstart.md +156 -0
- equiframe/docs/r_review.md +212 -0
- equiframe/docs/recipes.md +248 -0
- equiframe/docs/releasing.md +183 -0
- equiframe/docs/security.md +218 -0
- equiframe/docs/semantics.md +274 -0
- equiframe/docs/sqlite-snapshots.md +262 -0
- equiframe/docs/universal-sources.md +49 -0
- equiframe/docs/universalidad-cinco-pasos.md +107 -0
- equiframe/docs/upgrading.md +538 -0
- equiframe/duckdb_capture.py +410 -0
- equiframe/eda/__init__.py +36 -0
- equiframe/eda/__main__.py +3 -0
- equiframe/eda/_backend.py +612 -0
- equiframe/eda/_inventory.py +49 -0
- equiframe/eda/_paths.py +49 -0
- equiframe/eda/_report.py +291 -0
- equiframe/eda/api.py +535 -0
- equiframe/eda/cli.py +102 -0
- equiframe/eda/models.py +239 -0
- equiframe/errors.py +83 -0
- equiframe/evidence.py +375 -0
- equiframe/examples/integrations/eda_generic.py +60 -0
- equiframe/examples/integrations/outcome_gates.py +94 -0
- equiframe/examples/integrations/sqlite_snapshot.py +246 -0
- equiframe/examples/integrations/sqlite_snapshot_demo.py +226 -0
- equiframe/examples/integrations/universal_first_use.py +296 -0
- equiframe/examples/journeys/README.md +31 -0
- equiframe/examples/journeys/__init__.py +1 -0
- equiframe/examples/journeys/a_pandas_refactor.py +88 -0
- equiframe/examples/journeys/b_native_migration.py +181 -0
- equiframe/examples/journeys/c_retail_etl.py +460 -0
- equiframe/examples/journeys/common.py +136 -0
- equiframe/examples/journeys/d_financial_reconciliation.py +96 -0
- equiframe/examples/journeys/e_telemetry.py +135 -0
- equiframe/examples/journeys/f_data_science_preparation.py +84 -0
- equiframe/examples/journeys/g_external_snapshots.py +197 -0
- equiframe/examples/journeys/h_portable_ci.py +129 -0
- equiframe/examples/journeys/requirements.lock +249 -0
- equiframe/examples/journeys/verify_journeys.py +117 -0
- equiframe/examples/load_convert_compare.py +85 -0
- equiframe/execution.py +355 -0
- equiframe/execution_types.py +177 -0
- equiframe/ingestion.py +744 -0
- equiframe/local.py +126 -0
- equiframe/migration/__init__.py +94 -0
- equiframe/migration/_physical.py +8 -0
- equiframe/migration/backends/_timestamp_transport.py +109 -0
- equiframe/migration/backends/duckdb.py +651 -0
- equiframe/migration/backends/duckdb_decimal.py +125 -0
- equiframe/migration/backends/duckdb_time.py +87 -0
- equiframe/migration/backends/pandas.py +544 -0
- equiframe/migration/backends/polars.py +915 -0
- equiframe/migration/backends/polars_decimal.py +231 -0
- equiframe/migration/io.py +198 -0
- equiframe/migration/ports.py +96 -0
- equiframe/migration/runner.py +545 -0
- equiframe/migration/runner_types.py +405 -0
- equiframe/migration/types.py +354 -0
- equiframe/migration/validation.py +619 -0
- equiframe/normalization.py +358 -0
- equiframe/pipeline.py +274 -0
- equiframe/pipeline_types.py +247 -0
- equiframe/ports.py +39 -0
- equiframe/preprocessing.py +110 -0
- equiframe/profile_types.py +343 -0
- equiframe/profiles.py +447 -0
- equiframe/py.typed +0 -0
- equiframe/r_review.py +706 -0
- equiframe/reduction.py +518 -0
- equiframe/registry.py +121 -0
- equiframe/report_rendering.py +366 -0
- equiframe/report_types.py +294 -0
- equiframe/reporting.py +26 -0
- equiframe/rule_types.py +178 -0
- equiframe/rules.py +819 -0
- equiframe/schemas/r-execution-binding-1.0.schema.json +46 -0
- equiframe/schemas/r-review-binding-1.0.schema.json +28 -0
- equiframe/schemas/r-review-options-1.0.schema.json +38 -0
- equiframe/schemas/r-review-result-1.0.schema.json +207 -0
- equiframe/schemas/reviewed-decision-1.0.schema.json +3394 -0
- equiframe/serialization.py +152 -0
- equiframe/snapshot.py +296 -0
- equiframe/sources.py +283 -0
- equiframe/tables.py +459 -0
- equiframe/types.py +71 -0
- equiframe/values.py +252 -0
- equiframe/verification_request.py +35 -0
- equiframe/verify.py +799 -0
- equiframe/verify_types.py +255 -0
- equiframe/workflow.py +510 -0
- equiframe/workflow_request.py +38 -0
- equiframe-0.12.4.dist-info/METADATA +268 -0
- equiframe-0.12.4.dist-info/RECORD +170 -0
- equiframe-0.12.4.dist-info/WHEEL +4 -0
- equiframe-0.12.4.dist-info/entry_points.txt +3 -0
- equiframe-0.12.4.dist-info/licenses/LICENSE +21 -0
- equiframe-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md +42 -0
- equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/LICENSE +201 -0
- equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/PROVENANCE.md +18 -0
equiframe/docs/api.md
ADDED
|
@@ -0,0 +1,534 @@
|
|
|
1
|
+
# Public API reference
|
|
2
|
+
|
|
3
|
+
[Handbook](index.md) · [Choose an API](choosing-api.md) · [Semantics](semantics.md) · [Compatibility](compatibility.md)
|
|
4
|
+
|
|
5
|
+
Use `import equiframe as ef`. This reference describes the 0.12.0 implemented
|
|
6
|
+
entry points and their bounded stability contracts. Signatures below use Python notation;
|
|
7
|
+
`...` in a signature means the other documented optional arguments, not an
|
|
8
|
+
executable example. The exhaustive exported namespace is `ef.__all__` and its
|
|
9
|
+
per-name stability classifications are returned by `ef.get_stability_register()`.
|
|
10
|
+
A name being exported does not mean every input, platform or use is supported.
|
|
11
|
+
|
|
12
|
+
## Inspect the installed version, signatures and model fields
|
|
13
|
+
|
|
14
|
+
These commands work without a checkout and do not load optional engines merely
|
|
15
|
+
to inspect EquiFrame:
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
import inspect
|
|
19
|
+
import equiframe as ef
|
|
20
|
+
|
|
21
|
+
print(ef.__version__)
|
|
22
|
+
print(inspect.signature(ef.diff))
|
|
23
|
+
print(ef.ComparisonOptions.model_json_schema())
|
|
24
|
+
print(ef.get_compatibility_policy().model_dump_json(indent=2))
|
|
25
|
+
for entry in ef.get_stability_register():
|
|
26
|
+
print(entry.model_dump_json())
|
|
27
|
+
for capability in ef.get_adapter_capabilities():
|
|
28
|
+
print(capability.model_dump_json())
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Pydantic model fields are available through `Model.model_fields`, schemas through
|
|
32
|
+
`Model.model_json_schema()`, and JSON serialization through
|
|
33
|
+
`instance.model_dump_json()`. These helpers do not replace public cross-field
|
|
34
|
+
validation, artifact verification or declared compatibility rules. Models reject
|
|
35
|
+
extra fields and lossy coercions; Python collections are generally explicit
|
|
36
|
+
tuples where specified. Use `inspect.signature` for the complete installed
|
|
37
|
+
signature and Python `help` for entry-point descriptions.
|
|
38
|
+
|
|
39
|
+
## Explicit outcome objects
|
|
40
|
+
|
|
41
|
+
Operational result/check/decision models reject implicit truth testing. Use the
|
|
42
|
+
[exact inventory and migration guide](outcome-gates.md) to select each result's
|
|
43
|
+
state, completeness and acceptance fields; use `is not None` only for optional
|
|
44
|
+
presence. This is an intentional fail-closed Python behavior change. JSON fields,
|
|
45
|
+
format versions and fingerprints are unchanged.
|
|
46
|
+
|
|
47
|
+
## 1. Contracts, policies and serialization
|
|
48
|
+
|
|
49
|
+
| API | Behavior |
|
|
50
|
+
|---|---|
|
|
51
|
+
| `validate_contract(document) -> Contract` | Validate JSON text, UTF-8 bytes or a JSON-compatible mapping, including cross-field constraints |
|
|
52
|
+
| `read_contract(path) -> Contract` | Read a local file through the same validator |
|
|
53
|
+
| `write_contract(contract, path) -> None` | Revalidate and write canonical metadata bytes |
|
|
54
|
+
| `canonical_contract_bytes(contract) -> bytes` | Canonical metadata representation |
|
|
55
|
+
| `contract_fingerprint(contract) -> str` | SHA-256 of canonical contract metadata |
|
|
56
|
+
| `contract_json_schema() -> dict` | Structural Contract JSON Schema |
|
|
57
|
+
| `comparison_fingerprint(spec) -> str` | Identity of the complete `ComparisonSpec` |
|
|
58
|
+
|
|
59
|
+
Core model families:
|
|
60
|
+
|
|
61
|
+
- Schema: `Contract`, `TableSpec`, `TableSchema`, `ColumnSpec`, `KeySpec`,
|
|
62
|
+
`GlobalKeyScope`, `TableKeyScope`, `ColumnKeyScope`
|
|
63
|
+
- Logical types: `BooleanType`, `IntegerType`, `StringType`, `DecimalType`,
|
|
64
|
+
`FloatType`, `DateType`, `TimestampType`
|
|
65
|
+
- Policies: `NullPolicy`, `NaNPolicy`, `NumericPolicy`, `Tolerance`, `DatePolicy`,
|
|
66
|
+
`TimestampPolicy`, `IgnoredOrder`, `SignificantOrder`, `SortSpec`
|
|
67
|
+
- Declarative checks: `PreconditionSpec`, `RuleSpec`, `SchemaCheck`, `KeyCheck`,
|
|
68
|
+
`CardinalityCheck`, `JoinCheck`, `OrderCheck`, `AggregateCheck`
|
|
69
|
+
- Tagged values: `NullValue`, `NaNValue`, `InfinityValue`, `BooleanValue`,
|
|
70
|
+
`IntegerValue`, `StringValue`, `DecimalValue`, `FloatValue`, `DateValue`,
|
|
71
|
+
`TimestampValue`; `ScalarValue` is their declared union
|
|
72
|
+
- Artifact identity: `ArtifactRef`, `SchemaRef`, `Sha256`
|
|
73
|
+
|
|
74
|
+
Package version, `schema_version` and contract `revision` are independent.
|
|
75
|
+
Current public documents use their admitted format `1.0`; unknown versions,
|
|
76
|
+
duplicate JSON members, invalid UTF-8 and bare JSON NaN/Infinity are rejected.
|
|
77
|
+
`equiframe-metadata-json-v1` is EquiFrame's own canonicalization, not JCS/RFC 8785.
|
|
78
|
+
It retains defaults/nulls, array order, Unicode text and exact Decimal text.
|
|
79
|
+
Metadata fingerprints do not prove tabular equality or artifact authenticity.
|
|
80
|
+
|
|
81
|
+
## 2. Direct comparison
|
|
82
|
+
|
|
83
|
+
```text
|
|
84
|
+
diff(candidate, expected, *, contract=None, keys=None, policy=None,
|
|
85
|
+
reference=None, options=None, capture_options=None, work_dir=None,
|
|
86
|
+
cancellation=None) -> ComparisonResult
|
|
87
|
+
|
|
88
|
+
compare(left, right, *, contract, policy=None, options=None,
|
|
89
|
+
reference="peer", work_dir=None, cancellation=None) -> ComparisonResult
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
`diff` directly admits eager pandas/Polars DataFrames, Arrow tables and actual
|
|
93
|
+
`DuckDBPyRelation` objects. Passing a relation authorizes consuming that native
|
|
94
|
+
plan once per supplied input. Capture fetches at most `max_rows + 1`, rejects
|
|
95
|
+
excess rows instead of truncating, checks each Arrow batch against byte limits,
|
|
96
|
+
and detaches the result. The caller retains connection/session ownership; no SQL
|
|
97
|
+
string or live connection is inferred as a table. Cancellation is cooperative
|
|
98
|
+
before execution and between batches, including native first-fetch limitations.
|
|
99
|
+
Without a contract it infers conservatively from the right-hand schema and defaults to
|
|
100
|
+
`reference="expected"`. With a contract it also supports the materialized
|
|
101
|
+
`compare` input routes. Inference without a contract is limited to these admitted
|
|
102
|
+
native inputs and `ArrowInput`.
|
|
103
|
+
|
|
104
|
+
`compare` requires a `Contract`; inputs are `ArrowInput`, local `ArtifactRef`,
|
|
105
|
+
local Arrow IPC **file**/Parquet `Path`, `TableArtifact` or completed
|
|
106
|
+
`ExecutionResult`. It never runs transformations. Two ExecutionResults must have
|
|
107
|
+
compatible verified input identities/parameters; comparing their explicit
|
|
108
|
+
TableArtifacts instead makes no same-input execution claim.
|
|
109
|
+
|
|
110
|
+
Typed reusable declarations:
|
|
111
|
+
|
|
112
|
+
```text
|
|
113
|
+
bag_contract(columns, *, policy=None, multiplicity="preserve",
|
|
114
|
+
reference="expected", contract_id="diff", revision=1)
|
|
115
|
+
keyed_contract(columns, *, keys, incomplete="reject", policy=None,
|
|
116
|
+
multiplicity="preserve", reference="expected",
|
|
117
|
+
contract_id="diff", revision=1)
|
|
118
|
+
sequence_contract(columns, *, by, ties="preserve", policy=None,
|
|
119
|
+
multiplicity="preserve", reference="expected",
|
|
120
|
+
contract_id="diff", revision=1)
|
|
121
|
+
infer_diff_contract(reference_schema, *, keys=None, column_overrides=(),
|
|
122
|
+
policy=None, reference="expected")
|
|
123
|
+
read_diff_contract(result) -> DiffContract
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
These constructors return `DiffContract`, holding `.spec`, `.reference` and
|
|
127
|
+
field-level `.provenance`; `provenance_for(path)` uses a JSON Pointer.
|
|
128
|
+
`DiffContract.from_contract(contract, ...)` adapts an existing contract. Reuse
|
|
129
|
+
with `diff(..., contract=declaration)`; do not simultaneously override its keys,
|
|
130
|
+
policy or reference. `read_diff_contract` verifies the supplementary declaration's
|
|
131
|
+
format and links but does not rerun comparison or authenticate provenance labels.
|
|
132
|
+
|
|
133
|
+
`ComparisonResult` retains state, completeness, final/partial counts, captured
|
|
134
|
+
sources, applied policies, examples, diagnostics, contract/spec fingerprints and
|
|
135
|
+
the differences artifact. `read_comparison(path)` validates its manifest;
|
|
136
|
+
`iter_differences(result.differences)` verifies and iterates persisted records.
|
|
137
|
+
Differences expose code, key/position, column, typed values and multiplicities.
|
|
138
|
+
See [semantics](semantics.md) for key, bag and sequence guarantees.
|
|
139
|
+
|
|
140
|
+
`ComparisonOptions` controls batches, rows, work, SQLite/disk, approximate
|
|
141
|
+
matching, time and evidence. `DiffCaptureOptions` controls native capture
|
|
142
|
+
separately. They are cooperative budgets; [defaults and costs](performance.md)
|
|
143
|
+
are not hard resource quotas.
|
|
144
|
+
|
|
145
|
+
## 3. Explicit acceptance in tests and pipelines
|
|
146
|
+
|
|
147
|
+
```text
|
|
148
|
+
assert_equivalent(result, *, policy=None, evidence_root=None,
|
|
149
|
+
report_path=None) -> None
|
|
150
|
+
equivalence_decision(result, *, policy=None,
|
|
151
|
+
evidence_root=None) -> EquivalenceDecision
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
`assert_equivalent` returns only for accepted complete/final `observed_contract`
|
|
155
|
+
equivalence. `EquivalencePolicy` can require the reference role and reviewed
|
|
156
|
+
contract/spec hashes. Obtain those hashes from previously approved inputs;
|
|
157
|
+
copying them from an untrusted result is not a meaningful approval check.
|
|
158
|
+
|
|
159
|
+
Persistence is optional and requires **both**
|
|
160
|
+
`EquivalencePolicy(require_persisted_evidence=True)` and an explicit `Path`
|
|
161
|
+
`evidence_root`. It verifies the finalized manifest and difference evidence
|
|
162
|
+
within bounded permitted paths. Supplying only one is invalid. It does not
|
|
163
|
+
reexecute sources or certify full transformation reproducibility.
|
|
164
|
+
|
|
165
|
+
| Exception | Meaning |
|
|
166
|
+
|---|---|
|
|
167
|
+
| `EquivalenceDifferentError` | Complete differences |
|
|
168
|
+
| `EquivalenceNotEvaluableError` | Required judgment unavailable |
|
|
169
|
+
| `EquivalenceGuaranteeError` | Consumer's required hashes/reference not satisfied |
|
|
170
|
+
| `EquivalenceEvidenceError` | Required persisted evidence fails its checks |
|
|
171
|
+
| `EquivalenceInputError` | Invalid argument/model/policy configuration |
|
|
172
|
+
|
|
173
|
+
The first four derive from `EquivalenceAssertionError` (`AssertionError` and
|
|
174
|
+
`EquiFrameError`). Input errors derive from `ValueError` and `EquiFrameError`.
|
|
175
|
+
Use stable error codes, not prose message parsing. Decision `.accepted` is the
|
|
176
|
+
explicit machine-readable gate. `report_path` is only a caller-selected diagnostic
|
|
177
|
+
path; it is not opened or verified by the assertion.
|
|
178
|
+
|
|
179
|
+
## 4. Redacted comparison reports
|
|
180
|
+
|
|
181
|
+
```text
|
|
182
|
+
comparison_report(result, *, options=None, evidence_root=None,
|
|
183
|
+
declaration=None) -> ComparisonReport
|
|
184
|
+
write_comparison_report(result, directory, *, options=None,
|
|
185
|
+
evidence_root=None, declaration=None) -> ComparisonReport
|
|
186
|
+
read_comparison_report(path, *, max_bytes=2097152) -> ComparisonReport
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
`report.to_text()`, `report.to_html()` and notebook `_repr_html_()` share the same
|
|
190
|
+
filtered content. The public `render_comparison_text`, `render_comparison_html`
|
|
191
|
+
and `render_comparison_notebook` functions render that model. Default report
|
|
192
|
+
construction reads no files and hides values, names and paths.
|
|
193
|
+
|
|
194
|
+
`ReportOptions` selects `diagnostic`, `detail` or `reproduction`, explicit
|
|
195
|
+
`include_values`/`include_metadata`, `redact_columns`, examples and read/write
|
|
196
|
+
budgets. `evidence_root` explicitly permits bounded evidence inspection.
|
|
197
|
+
`require_exhaustive` requires both completed evaluation and verified complete
|
|
198
|
+
difference evidence, regardless of example limit. Required evidence can be
|
|
199
|
+
`not_satisfied` without changing the comparison's original verdict.
|
|
200
|
+
|
|
201
|
+
Writing requires a new directory and produces `report.json`, `report.txt`,
|
|
202
|
+
`report.html`, then `evidence-manifest.json`. Partial files without successful
|
|
203
|
+
publication are not a delivered report. `read_comparison_report` reads the view;
|
|
204
|
+
it does not verify a whole export's inventory. Reproduction requires explicit
|
|
205
|
+
value/metadata disclosure and supported complete materialized input captures;
|
|
206
|
+
it cannot reproduce arbitrary original SQL, ETL or services.
|
|
207
|
+
|
|
208
|
+
## 5. Explicit DuckDB capture
|
|
209
|
+
|
|
210
|
+
```text
|
|
211
|
+
capture_duckdb(sql, *, work_dir, name="result", connection=None,
|
|
212
|
+
exclusive_connection=False, parameters=None, options=None,
|
|
213
|
+
cancellation=None) -> TableArtifact
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
This stable, explicitly bounded entry point runs one trusted SELECT/WITH, checks native
|
|
217
|
+
types before Arrow and publishes detached Arrow IPC batches. `parameters` is a
|
|
218
|
+
mapping of bound values. It rejects unsupported native types rather than
|
|
219
|
+
silently losing distinctions. `DuckDBCaptureOptions` controls batch, row, byte
|
|
220
|
+
and cooperative time limits.
|
|
221
|
+
|
|
222
|
+
Without a supplied connection, it owns and closes a private connection. A borrowed
|
|
223
|
+
connection requires `exclusive_connection=True` and owner-disabled extension
|
|
224
|
+
autoload/autoinstall. It is not closed, configured, registered or committed by
|
|
225
|
+
this API; its result slot is consumed and an SQL error may require caller rollback.
|
|
226
|
+
The snapshot is reusable after the connection closes. Capture does not establish
|
|
227
|
+
a shared transaction across external systems and is not a SQL sandbox.
|
|
228
|
+
|
|
229
|
+
## 6. Registered native execution
|
|
230
|
+
|
|
231
|
+
```text
|
|
232
|
+
execute(plan, *, source, registry, adapter=None, artifact_store=None,
|
|
233
|
+
work_dir=None, cancellation=None) -> ExecutionResult
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
`ExecutionPlan` holds a contract, engine, registered transformation ID/revision,
|
|
237
|
+
tagged parameters and `ExecutionOptions`. `TransformationRegistry` is explicit
|
|
238
|
+
and per-instance; `.register(...)` does not replace an existing identical key.
|
|
239
|
+
|
|
240
|
+
- `PandasTransformation(id, callback, code_version)` receives named DataFrames
|
|
241
|
+
and `RunContext`, and returns a DataFrame
|
|
242
|
+
- `PolarsTransformation(id, callback, code_version)` receives native DataFrames
|
|
243
|
+
and can return a DataFrame or LazyFrame; collection is explicit in execution
|
|
244
|
+
- `DuckDBTransformation(id, sql, code_version)` registers trusted SELECT/WITH;
|
|
245
|
+
named `$parameter` values are bound from the plan
|
|
246
|
+
|
|
247
|
+
`code_version` is the caller's declaration, not an automatically computed hash
|
|
248
|
+
of callback behavior. Callbacks/plugins and dependencies may have external effects.
|
|
249
|
+
|
|
250
|
+
`SnapshotDataSource` consumes an explicitly pinned `SnapshotManifest` through
|
|
251
|
+
an injected `SnapshotClient`; both are now available from core alone. The
|
|
252
|
+
historical benchmark imports remain exact aliases. Metadata needs only base;
|
|
253
|
+
resolving data needs Arrow. See [extension ports](extending.md) for the complete
|
|
254
|
+
boundary, preservation rules and a runnable core-only example.
|
|
255
|
+
|
|
256
|
+
`DataSource` exposes `table_names` and `resolve(name, *, context) -> LoadedTable`.
|
|
257
|
+
`LocalDataSource` uses named `ArrowInput`, `PandasInput`, `PolarsInput` or local
|
|
258
|
+
artifact inputs. `EngineAdapter` exposes `engine`, `capabilities()` and native
|
|
259
|
+
`execute(transformation, inputs, context)`. `ArtifactStore` exposes
|
|
260
|
+
`put(path, *, media_type)` and `get(ref, destination)`; `LocalArtifactStore` verifies
|
|
261
|
+
local copies. `read_table(ref)` verifies size/hash before reading Arrow/Parquet.
|
|
262
|
+
|
|
263
|
+
`ExecutionResult.state` is `completed`, `invalid_input`, `failed`, `timeout`,
|
|
264
|
+
`resource_exhausted` or `cancelled`. **Completed only means execution and
|
|
265
|
+
materialization finished**. `execute` does not run preconditions, R, comparison
|
|
266
|
+
or anomaly detection. Operational failures inside a run preserve a typed problem;
|
|
267
|
+
invalid plans and evidence-creation/publication failures can raise directly.
|
|
268
|
+
`CancellationToken` and callback checkpoints are cooperative.
|
|
269
|
+
|
|
270
|
+
## 7. Rules, profiles, models and orchestration
|
|
271
|
+
|
|
272
|
+
| Family | Entry points and result types |
|
|
273
|
+
|---|---|
|
|
274
|
+
| Rules R | `evaluate_rules(execution, *, registry=None, options=None, work_dir=None, cancellation=None) -> RuleEvaluation`; `RuleRegistry`, `RuleOptions`, `read_rule_evaluation`, `iter_rule_findings` |
|
|
275
|
+
| Profiles | `profile_execution(execution, *, spec, options=None, work_dir=None, cancellation=None, conversions=()) -> ProfileResult`; `ProfileSpec`, `ProfileFeature`, `ProfileVector`, `read_profile` |
|
|
276
|
+
| Preprocessing | `fit_preprocessor(training, *, spec, empty_policy="drop")`; `transform_profiles(bundle, profiles) -> FeatureMatrix`; `read_preprocessor`, `write_preprocessor` |
|
|
277
|
+
| Fit | `fit_detectors(training, *, spec, config, support, empty_policy="drop") -> DetectorFit` |
|
|
278
|
+
| Score | `score_detectors(fit, profiles) -> DetectorScoreResult` |
|
|
279
|
+
| Calibrate | `calibrate_detectors(fit, calibration, *, alpha) -> DetectorBundle` |
|
|
280
|
+
| Infer | `detect(bundle, profiles) -> DetectionResult`; `detect_component(bundle, profiles, *, method)` for `isolation_forest` or `mad` |
|
|
281
|
+
| Hybrid | `combine_decisions(r, m) -> HybridDecision`; H is R OR M, without B |
|
|
282
|
+
| Persistence | `read_detector_bundle`, `write_detector_bundle`, `detector_fingerprint` |
|
|
283
|
+
| Pipeline | `run_pipeline(spec, *, source, registry, ...) -> PipelineResult`; `read_pipeline`, `inspect_evidence`, `verify_events`, `pipeline_summary` |
|
|
284
|
+
| Pipeline report | `render_report`, `write_report`; distinct from comparison reports |
|
|
285
|
+
| Reduction | `reduce_case(inputs, *, predicate, predicate_version, failure_signature, options=None, work_dir=None, cancellation=None)`; `read_reduction` |
|
|
286
|
+
|
|
287
|
+
Profiles, preprocessing and detector interfaces are stable within their declared
|
|
288
|
+
feature/version/support contracts. This does not promise statistical accuracy or
|
|
289
|
+
cross-version fitted-model parity. Standalone pipeline orchestration and
|
|
290
|
+
reduction remain experimental; shared pipeline models embedded in migration
|
|
291
|
+
retain their versioned format commitments.
|
|
292
|
+
Training/calibration wrappers (`NormalTrainingProfiles`,
|
|
293
|
+
`NormalCalibrationProfiles`) keep labels outside detector inputs; the benchmark
|
|
294
|
+
handles reviewed selection. An empty/unsupported component remains not evaluable.
|
|
295
|
+
Reduction invokes a **trusted** predicate with bounded row subsets, preserves
|
|
296
|
+
preconditions/signature and does not claim a global minimum.
|
|
297
|
+
|
|
298
|
+
## 8. Native migration, Verify and ETL
|
|
299
|
+
|
|
300
|
+
```text
|
|
301
|
+
check_migration(spec, *, compilers=None) -> tuple[CompilationCheck, CompilationCheck]
|
|
302
|
+
migrate(spec, *, source, work_dir=None, expected=None, compilers=None,
|
|
303
|
+
adapters=None, rule_registry=None, detector_bundle=None,
|
|
304
|
+
cancellation=None) -> MigrationResult
|
|
305
|
+
read_migration(path, *, verify=True) -> MigrationResult
|
|
306
|
+
|
|
307
|
+
verify(outputs, *, contract, expected=None, provenance=None, reviews=(),
|
|
308
|
+
request_review=False, require_expected=False, require_rules=False,
|
|
309
|
+
snapshots=None, plan_sha256=None, policy=None, options=None,
|
|
310
|
+
work_dir=None) -> VerificationResult
|
|
311
|
+
verify_migration(path, *, work_dir=None, provenance=None, reviews=(),
|
|
312
|
+
request_review=False) -> VerificationResult
|
|
313
|
+
|
|
314
|
+
ingest_file(path, *, spec, work_dir, context=None, batch_rows=4096) -> IngestionResult
|
|
315
|
+
run_workflow(spec, *, source, work_dir, expected=None,
|
|
316
|
+
cancellation=None) -> WorkflowResult
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
Migration models include `RelationalPlan`, `MigrationSpec`, `MigrationOptions`,
|
|
320
|
+
`MigrationRequest`, `MigrationResult`, `CompilerRegistry`, `PlanCompiler` and
|
|
321
|
+
`CompiledPlan`. `validate_ir`, `ir_fingerprint` and `native_compiler_registry`
|
|
322
|
+
cover the typed IR/compilers. The [semantics chapter](semantics.md) lists its
|
|
323
|
+
bounded operations and native limitations.
|
|
324
|
+
|
|
325
|
+
Migration `.state` is peer equivalence; `.verification` is `correct`, `divergent`,
|
|
326
|
+
`not_evaluable` or `not_requested` against the supplied expected output. `.complete`
|
|
327
|
+
and `.exit_code` reflect requested stages. Comparison with a declared expected
|
|
328
|
+
output alone does not prove its authorship, independence or correctness.
|
|
329
|
+
|
|
330
|
+
Verify keeps input validity, technical completion, integrity, agreement,
|
|
331
|
+
expected-output checks, rules, coverage and independent review separate.
|
|
332
|
+
`ExpectedProvenance` and `IndependentReviewEvidence` declare reviewed evidence;
|
|
333
|
+
JSON reviewer names are not authenticated identities. `read_verification` and
|
|
334
|
+
`verification_summary` inspect finalized records without rerunning ETL.
|
|
335
|
+
|
|
336
|
+
ETL models include `FileInput`, `IngestionSpec`, `IngestionResult`,
|
|
337
|
+
`IngestionConversionReceipt`, `WorkflowSpec`, `WorkflowStage`,
|
|
338
|
+
`WorkflowInputBinding`, `WorkflowRequest`, `WorkflowResult`, `Destination`,
|
|
339
|
+
`ParquetDestination` and `LoadReceipt`. Ingestion uses explicit CSV/JSON/JSONL
|
|
340
|
+
schema/grain/path rules and retains original sources/quarantine receipts.
|
|
341
|
+
[CSV/JSONL batching](ingestion.md) limits accepted-value arrays, with complete
|
|
342
|
+
receipt tuples retained; full JSON and execution inputs still materialize.
|
|
343
|
+
`FileInput(path, spec, batch_rows=4096)` accepts the same positive integer knob.
|
|
344
|
+
Workflows fix inputs and native checkpoints, retain failed/skipped stages and
|
|
345
|
+
have no resume guarantee. Local Parquet publication is a separate verified
|
|
346
|
+
workflow; it does not implement a shared production data gateway.
|
|
347
|
+
|
|
348
|
+
## 9. Portable evidence
|
|
349
|
+
|
|
350
|
+
```text
|
|
351
|
+
export_bundle(source, destination) -> EvidenceBundle
|
|
352
|
+
import_bundle(source, destination, *, expected_sha256=None) -> EvidenceBundle
|
|
353
|
+
read_bundle(path, *, expected_sha256=None) -> EvidenceBundle
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
`inspect_bundle` provides a summary of the verified inventory. Supported sources
|
|
357
|
+
are finalized migration, verification and workflow evidence. The bundle retains
|
|
358
|
+
historical bytes and resolves old references only to captured members with
|
|
359
|
+
matching size/hash; it does not execute callbacks. An independently trusted
|
|
360
|
+
`expected_sha256` pins the bundle manifest. Absolute local paths elsewhere do
|
|
361
|
+
not become portable automatically. Benchmark study bundles are a separate API.
|
|
362
|
+
|
|
363
|
+
## 10. Command-line interface
|
|
364
|
+
|
|
365
|
+
Use `python -m equiframe --help` or the installed `equiframe` command. Subcommand
|
|
366
|
+
`--help` is the authority for exact installed parameters. Examples below show
|
|
367
|
+
request **shapes**; supply your own validated JSON and local artifacts:
|
|
368
|
+
|
|
369
|
+
```text
|
|
370
|
+
equiframe contract validate contract.json
|
|
371
|
+
equiframe compare comparison-spec.json candidate.arrow expected.arrow --expected --output comparisons
|
|
372
|
+
equiframe migration validate migration-spec.json
|
|
373
|
+
equiframe migration run migration-request.json --output migrations
|
|
374
|
+
equiframe migration verify migrations/migration_ID
|
|
375
|
+
equiframe verify validate verification-request.json
|
|
376
|
+
equiframe verify run verification-request.json --output verifications
|
|
377
|
+
equiframe verify migration migrations/migration_ID --output verifications
|
|
378
|
+
equiframe verify inspect verifications/verification_ID
|
|
379
|
+
equiframe ingestion run ingestion-spec.json input.csv --output ingestion
|
|
380
|
+
equiframe dataset inspect ventas.json
|
|
381
|
+
equiframe dataset load ventas.json --target polars
|
|
382
|
+
equiframe dataset load libro.xlsx --sheet Ventas --target duckdb
|
|
383
|
+
equiframe workflow validate workflow-request.json
|
|
384
|
+
equiframe workflow run workflow-request.json --output workflows
|
|
385
|
+
equiframe workflow inspect workflows/workflow_ID
|
|
386
|
+
equiframe bundle export migrations/migration_ID portable-bundle
|
|
387
|
+
equiframe bundle verify portable-bundle
|
|
388
|
+
equiframe bundle import portable-bundle imported-bundle
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
Pipeline `run`, `report`, `evidence verify` and `reduce` are also available;
|
|
392
|
+
explicit factory references load **trusted code** and are not safe for arbitrary
|
|
393
|
+
untrusted modules. `report` is the pipeline reporter, not a comparison report CLI.
|
|
394
|
+
|
|
395
|
+
| Command category | Exit meaning |
|
|
396
|
+
|---|---|
|
|
397
|
+
| `compare` | 0 accepted complete equivalence; 1 complete difference; 2 invalid input/configuration; 3 not evaluable/failure |
|
|
398
|
+
| Operational `run`, migration/workflow run, Verify run/migration | 0/1/2/3 distinguish successful requested checks, alert/difference/failure of checks, invalid input and unavailable/incomplete work as documented by that command |
|
|
399
|
+
| Integrity `inspect`/`verify` | 0 intact evidence, including a historically failed/partial run; 3 invalid evidence; argument errors may be 2 |
|
|
400
|
+
| `contract validate` | 0 valid metadata; 2 invalid arguments/JSON/contract/version; 3 I/O failure |
|
|
401
|
+
| `ingestion run` | 0 complete; 1 quarantined/rejected records; 2 invalid input; 3 technical failure |
|
|
402
|
+
| `dataset load` / `dataset inspect` | 0 complete bounded reading and requested conversion; 2 invalid arguments/configuration; 3 unavailable dependency or technical failure. Zero is not semantic equivalence |
|
|
403
|
+
| `reduce` | 0 reduced/unchanged; 2 invalid input; 3 failure/limit, not a global-minimum claim |
|
|
404
|
+
|
|
405
|
+
Operational failures write the original machine-readable code on the first
|
|
406
|
+
stderr line, followed by one fixed-text `hint[category]: ...` line. Categories
|
|
407
|
+
distinguish missing paths, existing outputs, permissions, invalid documents,
|
|
408
|
+
optional dependencies, unsupported capabilities and resource limits. Scripts
|
|
409
|
+
should use the exit code and first line, rather than match the whole message.
|
|
410
|
+
Hints do not include exception text, input values or local paths. The existing
|
|
411
|
+
`contract validate` command retains its structured validation locations and
|
|
412
|
+
codes; completed commands retain their existing stdout format.
|
|
413
|
+
Dataset reading/configuration errors may add explanatory text and, for an
|
|
414
|
+
unresolved choice, a JSON object with `required_option` and `choices` after the
|
|
415
|
+
stable code and hint. Scripts should not treat those later lines as a new
|
|
416
|
+
success/error protocol or parse their human-readable prose.
|
|
417
|
+
|
|
418
|
+
Never treat every command's zero as data equivalence. Never suppress nonzero exit
|
|
419
|
+
codes in CI without preserving the underlying failure. `--help` normally exits 0.
|
|
420
|
+
The historical `equiframe.benchmark` helper is **unsupported and quarantined**:
|
|
421
|
+
its helpers raise `UnsupportedBenchmarkError`; CLI exits 2 except help. Use the
|
|
422
|
+
separate experimental package only within its documented scope.
|
|
423
|
+
|
|
424
|
+
## 11. File discovery, Pandas loading and data conversion
|
|
425
|
+
|
|
426
|
+
```text
|
|
427
|
+
load_dataset(source, *, target="pandas", options=None) -> DatasetResult
|
|
428
|
+
inspect_dataset(source, *, options=None) -> DatasetInfo
|
|
429
|
+
convert_dataset(dataset, *, target) -> DatasetResult
|
|
430
|
+
```
|
|
431
|
+
|
|
432
|
+
These stable, capability-bounded APIs read local JSON/JSONL, Excel, TXT, XML, CSV, Parquet and
|
|
433
|
+
GeoJSON files into a real Pandas DataFrame. `target="polars"` or `"duckdb"`
|
|
434
|
+
converts that same DataFrame to the requested native engine. This is data
|
|
435
|
+
conversion; it does not translate transformation code or simulate independent
|
|
436
|
+
native executions. Full conversion readback rejects representation loss.
|
|
437
|
+
|
|
438
|
+
`DatasetReadOptions` exposes optional format/encoding, sheet, record path,
|
|
439
|
+
delimiter/header, text mode, conservative type inference, per-column explicit
|
|
440
|
+
types and admission budgets. `DatasetChoiceRequired.option` and `.choices`
|
|
441
|
+
identify an ambiguity the caller must resolve. No arbitrary file reader,
|
|
442
|
+
callback or dependency installation is executed from configuration.
|
|
443
|
+
|
|
444
|
+
`DatasetResult.pandas` is the original loaded frame; `.frame` is the native
|
|
445
|
+
destination; `.target` identifies it; `.info` describes rows, columns and
|
|
446
|
+
inference. `DatasetInfo.to_dict()` omits original metadata unless
|
|
447
|
+
`include_metadata=True` is requested. DuckDB uses a private in-memory connection
|
|
448
|
+
owned by the result: keep it open while using the relation and release it with
|
|
449
|
+
`close()` or `with`. `to_parquet(path)` exports the loaded Pandas data to a new
|
|
450
|
+
file; it does not export later operations on the destination frame. Publication
|
|
451
|
+
uses a synced sibling temporary file and an exclusive same-filesystem hard link;
|
|
452
|
+
an unsupported filesystem fails without falling back to a partial copy. See the
|
|
453
|
+
file guide for cleanup and recovery if an error occurs after publication.
|
|
454
|
+
|
|
455
|
+
`DatasetSource(uri, format, sha256, size_bytes, read_options_json)` records the
|
|
456
|
+
single byte snapshot actually consumed and the effective reading options;
|
|
457
|
+
`.read_options` exposes the decoded options. `DatasetProvenance` combines that
|
|
458
|
+
source, `pandas_sha256`, `target` and `target_sha256`. The result and its `info`
|
|
459
|
+
expose `.provenance`; `info.to_dict()` includes it only with
|
|
460
|
+
`include_provenance=True`, separately from original metadata. Conversion/export
|
|
461
|
+
reject a modified original Pandas frame instead of reusing an outdated source
|
|
462
|
+
identity. Hashes identify observed content and options, not trusted authorship.
|
|
463
|
+
|
|
464
|
+
`equiframe dataset load file` asks for the target in the same interactive
|
|
465
|
+
invocation. Noninteractive callers pass `--target pandas|polars|duckdb`; reading
|
|
466
|
+
choices can also be supplied explicitly. `dataset inspect` only describes the
|
|
467
|
+
loaded data. Without `--output` the CLI object exists only for the duration of
|
|
468
|
+
that command; Python callers receive the objects directly.
|
|
469
|
+
|
|
470
|
+
See [file loading and conversion](lectura-conversion.md) for exact supported
|
|
471
|
+
layouts, Decimal/identifier preservation, explicit `json` representation,
|
|
472
|
+
XML/Excel boundaries, resources and dependency extras.
|
|
473
|
+
|
|
474
|
+
## 12. Optional independent review in the R language
|
|
475
|
+
|
|
476
|
+
```text
|
|
477
|
+
review_with_r(left, right, *, contract, comparison=None, options=None,
|
|
478
|
+
cancellation=None) -> RReviewResult
|
|
479
|
+
finalize_with_r(rules, model, *, left, right, contract, comparison=None,
|
|
480
|
+
options=None, cancellation=None) -> ReviewedDecision
|
|
481
|
+
finalize_execution_with_r(execution, rules, profile, detection, *, comparison,
|
|
482
|
+
options=None, cancellation=None) -> ReviewedDecision
|
|
483
|
+
```
|
|
484
|
+
|
|
485
|
+
The optional reviewer implements exact comparison a second time in base R
|
|
486
|
+
over the complete observed tables and admitted contract. It does not consume
|
|
487
|
+
Python's verdict as its expected answer, retrain Isolation Forest, replace
|
|
488
|
+
operational rules R or establish an independent business oracle O.
|
|
489
|
+
Its interface and explicit unavailable/error states belong to the stable
|
|
490
|
+
[compatibility contract](compatibility.md); its admitted type/policy subset
|
|
491
|
+
remains narrower than arbitrary comparison contracts.
|
|
492
|
+
|
|
493
|
+
`RReviewOptions` configures a local, user-installed `Rscript`, cooperative
|
|
494
|
+
timeout and capture/transport budgets. `RReviewResult` exposes `.state`,
|
|
495
|
+
`.complete`, `.reason`, row accounting, source/specification/transport hashes
|
|
496
|
+
and the observed R version. Missing R, unsupported policy, cancellation, timeout
|
|
497
|
+
and technical failure produce `not_evaluable`; invalid API options/contracts
|
|
498
|
+
raise their validation errors. There is no automatic R/package installation.
|
|
499
|
+
|
|
500
|
+
`finalize_with_r` preserves H = rules OR Isolation Forest and all alerts from
|
|
501
|
+
the optional previous Python comparison. Its `ReviewedDecision` keeps the
|
|
502
|
+
components, comparator disagreement and explicit final `.result`/`.accepted`.
|
|
503
|
+
An unavailable required component cannot become acceptance, and equivalent R
|
|
504
|
+
outputs cannot erase an existing rule/model/comparison alert. Both result
|
|
505
|
+
models reject implicit truth testing.
|
|
506
|
+
|
|
507
|
+
Use `finalize_execution_with_r` for the execution-bound route: pass the actual
|
|
508
|
+
`ExecutionResult`, its `RuleEvaluation` and `ProfileResult`, and a
|
|
509
|
+
`ComponentDetectionResult` obtained with
|
|
510
|
+
`detect_component(bundle, (profile.vector,), method="isolation_forest")`.
|
|
511
|
+
It checks execution/contract/artifact/profile identities and IF association,
|
|
512
|
+
then reads the comparison's verified artifacts directly. Invalid associations
|
|
513
|
+
raise validation errors before R starts. `.execution_binding` records the
|
|
514
|
+
`RExecutionBinding` identities of the execution, profile, detection and output.
|
|
515
|
+
|
|
516
|
+
When `comparison` is supplied to `review_with_r` or the low-level finalizer,
|
|
517
|
+
its actual artifacts are verified and its full observed inputs must match the
|
|
518
|
+
transport sent to R. `RReviewResult.binding` records `RReviewBinding` with the
|
|
519
|
+
comparison ID and both artifact hashes. A standalone review without a Python
|
|
520
|
+
comparison does not invent that binding. `finalize_with_r` remains low-level:
|
|
521
|
+
the caller owns the association of isolated `ComponentDecision` inputs.
|
|
522
|
+
|
|
523
|
+
See [independent R review](r_review.md) for input binding, policy restrictions,
|
|
524
|
+
resource ownership and interpretation of final acceptance. This is a Python
|
|
525
|
+
API; there is no separate R-review CLI command.
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
## Native IR additions in 0.12.0a6
|
|
529
|
+
|
|
530
|
+
See the [executable contract reference](migration-ir.md) for `EquiJoin.cardinality`,
|
|
531
|
+
right/full joins, `Deduplicate.keep="first"` and its deterministic `by`,
|
|
532
|
+
`DecimalRound`, Decimal `Arithmetic` multiply/divide, and `LocalizeTimestamp`.
|
|
533
|
+
These expression models are stable public exports in 0.12.0. Engine-specific
|
|
534
|
+
rejections remain part of the capability contract, not fallback behavior.
|