equiframe 0.12.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- equiframe/__init__.py +832 -0
- equiframe/__main__.py +3 -0
- equiframe/_base.py +84 -0
- equiframe/_cli_support.py +125 -0
- equiframe/_comparison_io.py +161 -0
- equiframe/_dataset_provenance.py +146 -0
- equiframe/_decimal.py +22 -0
- equiframe/_diff_capture.py +433 -0
- equiframe/_doctor.py +123 -0
- equiframe/_document_io.py +61 -0
- equiframe/_documents.py +79 -0
- equiframe/_duckdb_transport.py +149 -0
- equiframe/_error_reporting.py +52 -0
- equiframe/_first_use.py +102 -0
- equiframe/_path_compat.py +26 -0
- equiframe/_physical.py +133 -0
- equiframe/_r_review.R +308 -0
- equiframe/_regular_io.py +167 -0
- equiframe/_report_format_v1.py +298 -0
- equiframe/_report_io.py +591 -0
- equiframe/_stability_inventory.py +1421 -0
- equiframe/_timezones.py +257 -0
- equiframe/_vendor/cyclonedx/LICENSE +201 -0
- equiframe/_vendor/cyclonedx/PROVENANCE.md +18 -0
- equiframe/_vendor/cyclonedx/bom-1.6.schema.json +5673 -0
- equiframe/_vendor/cyclonedx/jsf-0.82.schema.json +240 -0
- equiframe/_vendor/cyclonedx/spdx.schema.json +712 -0
- equiframe/adapters.py +454 -0
- equiframe/assertion_types.py +122 -0
- equiframe/assertions.py +271 -0
- equiframe/benchmark.py +101 -0
- equiframe/bundle.py +240 -0
- equiframe/cli.py +693 -0
- equiframe/comparison.py +937 -0
- equiframe/comparison_report.py +660 -0
- equiframe/comparison_types.py +257 -0
- equiframe/compatibility.py +546 -0
- equiframe/contracts.py +879 -0
- equiframe/datasets.py +1301 -0
- equiframe/decision.py +34 -0
- equiframe/destinations.py +295 -0
- equiframe/detector_types.py +431 -0
- equiframe/detectors.py +389 -0
- equiframe/diff.py +192 -0
- equiframe/diff_contracts.py +523 -0
- equiframe/docs/adoption.md +329 -0
- equiframe/docs/api.md +534 -0
- equiframe/docs/arquitecto.md +143 -0
- equiframe/docs/choosing-api.md +84 -0
- equiframe/docs/cientifico.md +180 -0
- equiframe/docs/compatibility.md +254 -0
- equiframe/docs/contributing.md +171 -0
- equiframe/docs/dependencies.md +237 -0
- equiframe/docs/eda.md +166 -0
- equiframe/docs/extending.md +158 -0
- equiframe/docs/fast-ci.md +52 -0
- equiframe/docs/first-use.md +78 -0
- equiframe/docs/index.md +142 -0
- equiframe/docs/ingestion.md +145 -0
- equiframe/docs/inicio.md +129 -0
- equiframe/docs/installed-private.md +97 -0
- equiframe/docs/journeys.md +362 -0
- equiframe/docs/lectura-conversion.md +211 -0
- equiframe/docs/licensing.md +17 -0
- equiframe/docs/maintenance.md +233 -0
- equiframe/docs/migration-ir.md +333 -0
- equiframe/docs/option-audit.md +45 -0
- equiframe/docs/outcome-gates.md +134 -0
- equiframe/docs/performance.md +263 -0
- equiframe/docs/piloto-simulado.md +56 -0
- equiframe/docs/quickstart.md +156 -0
- equiframe/docs/r_review.md +212 -0
- equiframe/docs/recipes.md +248 -0
- equiframe/docs/releasing.md +183 -0
- equiframe/docs/security.md +218 -0
- equiframe/docs/semantics.md +274 -0
- equiframe/docs/sqlite-snapshots.md +262 -0
- equiframe/docs/universal-sources.md +49 -0
- equiframe/docs/universalidad-cinco-pasos.md +107 -0
- equiframe/docs/upgrading.md +538 -0
- equiframe/duckdb_capture.py +410 -0
- equiframe/eda/__init__.py +36 -0
- equiframe/eda/__main__.py +3 -0
- equiframe/eda/_backend.py +612 -0
- equiframe/eda/_inventory.py +49 -0
- equiframe/eda/_paths.py +49 -0
- equiframe/eda/_report.py +291 -0
- equiframe/eda/api.py +535 -0
- equiframe/eda/cli.py +102 -0
- equiframe/eda/models.py +239 -0
- equiframe/errors.py +83 -0
- equiframe/evidence.py +375 -0
- equiframe/examples/integrations/eda_generic.py +60 -0
- equiframe/examples/integrations/outcome_gates.py +94 -0
- equiframe/examples/integrations/sqlite_snapshot.py +246 -0
- equiframe/examples/integrations/sqlite_snapshot_demo.py +226 -0
- equiframe/examples/integrations/universal_first_use.py +296 -0
- equiframe/examples/journeys/README.md +31 -0
- equiframe/examples/journeys/__init__.py +1 -0
- equiframe/examples/journeys/a_pandas_refactor.py +88 -0
- equiframe/examples/journeys/b_native_migration.py +181 -0
- equiframe/examples/journeys/c_retail_etl.py +460 -0
- equiframe/examples/journeys/common.py +136 -0
- equiframe/examples/journeys/d_financial_reconciliation.py +96 -0
- equiframe/examples/journeys/e_telemetry.py +135 -0
- equiframe/examples/journeys/f_data_science_preparation.py +84 -0
- equiframe/examples/journeys/g_external_snapshots.py +197 -0
- equiframe/examples/journeys/h_portable_ci.py +129 -0
- equiframe/examples/journeys/requirements.lock +249 -0
- equiframe/examples/journeys/verify_journeys.py +117 -0
- equiframe/examples/load_convert_compare.py +85 -0
- equiframe/execution.py +355 -0
- equiframe/execution_types.py +177 -0
- equiframe/ingestion.py +744 -0
- equiframe/local.py +126 -0
- equiframe/migration/__init__.py +94 -0
- equiframe/migration/_physical.py +8 -0
- equiframe/migration/backends/_timestamp_transport.py +109 -0
- equiframe/migration/backends/duckdb.py +651 -0
- equiframe/migration/backends/duckdb_decimal.py +125 -0
- equiframe/migration/backends/duckdb_time.py +87 -0
- equiframe/migration/backends/pandas.py +544 -0
- equiframe/migration/backends/polars.py +915 -0
- equiframe/migration/backends/polars_decimal.py +231 -0
- equiframe/migration/io.py +198 -0
- equiframe/migration/ports.py +96 -0
- equiframe/migration/runner.py +545 -0
- equiframe/migration/runner_types.py +405 -0
- equiframe/migration/types.py +354 -0
- equiframe/migration/validation.py +619 -0
- equiframe/normalization.py +358 -0
- equiframe/pipeline.py +274 -0
- equiframe/pipeline_types.py +247 -0
- equiframe/ports.py +39 -0
- equiframe/preprocessing.py +110 -0
- equiframe/profile_types.py +343 -0
- equiframe/profiles.py +447 -0
- equiframe/py.typed +0 -0
- equiframe/r_review.py +706 -0
- equiframe/reduction.py +518 -0
- equiframe/registry.py +121 -0
- equiframe/report_rendering.py +366 -0
- equiframe/report_types.py +294 -0
- equiframe/reporting.py +26 -0
- equiframe/rule_types.py +178 -0
- equiframe/rules.py +819 -0
- equiframe/schemas/r-execution-binding-1.0.schema.json +46 -0
- equiframe/schemas/r-review-binding-1.0.schema.json +28 -0
- equiframe/schemas/r-review-options-1.0.schema.json +38 -0
- equiframe/schemas/r-review-result-1.0.schema.json +207 -0
- equiframe/schemas/reviewed-decision-1.0.schema.json +3394 -0
- equiframe/serialization.py +152 -0
- equiframe/snapshot.py +296 -0
- equiframe/sources.py +283 -0
- equiframe/tables.py +459 -0
- equiframe/types.py +71 -0
- equiframe/values.py +252 -0
- equiframe/verification_request.py +35 -0
- equiframe/verify.py +799 -0
- equiframe/verify_types.py +255 -0
- equiframe/workflow.py +510 -0
- equiframe/workflow_request.py +38 -0
- equiframe-0.12.4.dist-info/METADATA +268 -0
- equiframe-0.12.4.dist-info/RECORD +170 -0
- equiframe-0.12.4.dist-info/WHEEL +4 -0
- equiframe-0.12.4.dist-info/entry_points.txt +3 -0
- equiframe-0.12.4.dist-info/licenses/LICENSE +21 -0
- equiframe-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md +42 -0
- equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/LICENSE +201 -0
- equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/PROVENANCE.md +18 -0
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# Security boundaries and responsible use
|
|
2
|
+
|
|
3
|
+
EquiFrame is a local Python library, not a sandbox or a hosted service.
|
|
4
|
+
The `0.12.4` release must satisfy the [release gates](releasing.md) before
|
|
5
|
+
publication; a version string alone does not establish acceptance. The historical
|
|
6
|
+
STEP9 assessment remains **incomplete** and is not relabelled by a later release.
|
|
7
|
+
Ordinary source review, dependency advisories and security regressions have their
|
|
8
|
+
own explicit scope. There is no completed security certification, universal
|
|
9
|
+
safety claim or assertion that the code has no vulnerabilities.
|
|
10
|
+
Dependency advisory results, when available, describe a dated snapshot and the
|
|
11
|
+
exact resolved versions; they do not replace a source review.
|
|
12
|
+
|
|
13
|
+
## Threat model and operating assumptions
|
|
14
|
+
|
|
15
|
+
Assets include source datasets, SQL and parameters, local paths, captured rows,
|
|
16
|
+
contracts, models, output reports, credentials in the host process and available
|
|
17
|
+
CPU, memory and disk. A consumer may receive damaged or malicious files from a
|
|
18
|
+
third party. A local user who can rewrite the evidence directory while it is
|
|
19
|
+
being checked is a stronger adversary than an immutable downloaded artifact.
|
|
20
|
+
Do not place trusted evidence in a directory writable by untrusted users.
|
|
21
|
+
|
|
22
|
+
| Boundary | Intended controls and residual responsibility |
|
|
23
|
+
|---|---|
|
|
24
|
+
| Contracts and JSON reports | Strict versioned data models, closed format inventories and validation reject many malformed inputs. Use only supported readers; unknown versions fail closed. Format validation does not guarantee a hard CPU/memory bound for every reader |
|
|
25
|
+
| Evidence and bundles | Hashes, sizes and closed inventories detect inconsistency. Hashes do not authenticate the producer or prove semantic correctness. Fix the expected hash through a trusted channel; inspect in an isolated working directory |
|
|
26
|
+
| HTML/text/notebook output | Common report views escape text and minimize values, names and paths by default. An explicit disclosure option can expose sensitive information. Inspect the chosen report before sharing; raw evidence is separate |
|
|
27
|
+
| Local paths and files | Keep controlled, private work/output roots. A filename is not proof of a regular file; device files, pipes, links, races and hostile mounts require host-level controls. Do not feed special files to readers |
|
|
28
|
+
| SQL and callbacks | Execute only trusted SQL, callbacks, adapters and compilers. Parameters protect value binding; they do not make SQL safe to run. Queries can read local files, access networks or invoke engine capabilities. JSON readers do not authorize executing referenced code |
|
|
29
|
+
| Plugins and models | Registration is explicit. Do not install or execute unknown plugins or deserialize executable Python objects. Detector persistence uses declared data formats; never substitute arbitrary pickle/joblib input |
|
|
30
|
+
| Language-R review | Only the shipped reviewer and a trusted, caller-selected `Rscript` executable may run. Review uses a private temporary directory and a subprocess timeout. It does not sandbox an arbitrary R program or make a semantic verdict a security verdict |
|
|
31
|
+
| Resources | Core budgets are cooperative and scoped to their operation. Capture, parsing, native allocation and custom callbacks may allocate or run before a checkpoint. Use OS/process/container quotas for hostile workloads; constrain disk, CPU, memory, threads and networking |
|
|
32
|
+
| Dependencies and build tools | Verify exact wheel/sdist hashes and the selected dependency profile. Keep build/dev tooling separate from runtime. Installation/building itself executes trusted tooling; an SBOM is an inventory, not a trust grant |
|
|
33
|
+
|
|
34
|
+
Read-only SQL is not a security perimeter. Explicit DuckDB capture disables
|
|
35
|
+
extension autoload/autoinstall on its private connection; a caller-owned
|
|
36
|
+
connection must already meet the declared conditions. These controls do not
|
|
37
|
+
make arbitrary third-party queries safe. Consistency across external files or
|
|
38
|
+
systems is not established merely by opening a database transaction.
|
|
39
|
+
|
|
40
|
+
## Local operation, networking and data disclosure
|
|
41
|
+
|
|
42
|
+
The product's default integration is local; it does not configure a telemetry
|
|
43
|
+
endpoint, background upload service or analytics opt-in. This is not a promise
|
|
44
|
+
of network isolation. Downloading dependencies, querying advisory services,
|
|
45
|
+
injected clients, user callbacks, database queries and external engine features
|
|
46
|
+
can use the network. Those are separate from rendering a stored report.
|
|
47
|
+
|
|
48
|
+
Use an isolated environment with networking denied when you require offline
|
|
49
|
+
processing, and provision its dependencies beforehand. Do not put credentials
|
|
50
|
+
in contracts, SQL text, data values, filenames, command arguments or diagnostics.
|
|
51
|
+
Error causes and third-party exception objects can retain details even when the
|
|
52
|
+
normal report view is minimized. Do not upload complete runs as bug reports.
|
|
53
|
+
|
|
54
|
+
Snapshot, spill, SQLite and raw evidence files can contain unredacted values.
|
|
55
|
+
A minimized HTML report does not sanitize these files. Use restrictive directory
|
|
56
|
+
permissions, encrypted storage if required, and an explicit retention policy.
|
|
57
|
+
Deleting a file is not a secure-erasure guarantee. Hashing low-entropy personal
|
|
58
|
+
values is not anonymization. Choose sharing and retention separately from the
|
|
59
|
+
comparison contract.
|
|
60
|
+
|
|
61
|
+
## Known review limits and reporting
|
|
62
|
+
|
|
63
|
+
The historical review observed a special-file availability risk in a bundle reader.
|
|
64
|
+
The A14 development follow-up reproduced blocking on a POSIX FIFO and added
|
|
65
|
+
regular-file checks to the bundle, JSON, event and file-description readers.
|
|
66
|
+
Capture budgets still require operation-specific interpretation; per-comparison
|
|
67
|
+
limits must not be represented as global quotas. The follow-up does not complete
|
|
68
|
+
or retroactively amend the historical assessment.
|
|
69
|
+
|
|
70
|
+
Hernán Troya (`htroya`) is responsible for maintenance, releases and security
|
|
71
|
+
response. There is no response-time SLA. Report suspected vulnerabilities through
|
|
72
|
+
the [private GitHub reporting channel](https://github.com/htroya/equiframe/security/advisories/new)
|
|
73
|
+
of the project's public repository when that route is available. A GitHub
|
|
74
|
+
account is required. For each release, the maintainer must enable this route
|
|
75
|
+
and the public issue channel and verify their availability. Use
|
|
76
|
+
[public issues](https://github.com/htroya/equiframe/issues) for non-sensitive
|
|
77
|
+
bugs and questions. If private intake cannot be verified, retain sensitive
|
|
78
|
+
details locally until confidential intake can be arranged.
|
|
79
|
+
|
|
80
|
+
Provide a sanitized reproducer, exact version and artifact hash, dependency and
|
|
81
|
+
runtime versions, expected behaviour and impact. Do not post credentials,
|
|
82
|
+
personal datasets or a sensitive exploit in a public issue. Reporting a concern
|
|
83
|
+
does not authorize accessing another user's data or running destructive tests.
|
|
84
|
+
|
|
85
|
+
[Release and withdrawal process](releasing.md) · [Dependency evidence](dependencies.md)
|
|
86
|
+
|
|
87
|
+
## Maintainer checks
|
|
88
|
+
|
|
89
|
+
Run security tools in a separate, disposable tooling environment. Pin the tool
|
|
90
|
+
versions, check downloaded hashes, fix the source commit and record the source
|
|
91
|
+
inventory. Scan the current tracked tree and any proposed additions, including
|
|
92
|
+
tests and documentation; Git history needs its own explicit scan when historical
|
|
93
|
+
exposure is in scope. A current-tree scan does not inspect every historical blob.
|
|
94
|
+
Keep raw reports local and private: scanner output may repeat the secret it found.
|
|
95
|
+
Share only sanitized locations, finding identifiers, dispositions and exit codes.
|
|
96
|
+
Delete temporary snapshots and reports after review. Do not overwrite dated
|
|
97
|
+
evidence or transfer a result to different release bytes.
|
|
98
|
+
|
|
99
|
+
For TruffleHog, use a local filesystem snapshot with `--no-verification`,
|
|
100
|
+
`--no-update`, `--no-ignore-tag`, `--fail`, `--fail-on-scan-errors` and JSON output.
|
|
101
|
+
Include verified, unknown, unverified and filtered-unverified result classes.
|
|
102
|
+
Disabling verification prevents checking candidate credentials against providers;
|
|
103
|
+
it does not establish whether a detected credential is valid. Exit 183 denotes
|
|
104
|
+
findings and must remain visible even when review determines they are false
|
|
105
|
+
positives. Exit 1 or a missing completion record must be investigated as a failed
|
|
106
|
+
scan. Review findings by their exact location and source context; do not suppress
|
|
107
|
+
all hexadecimal strings, URL credentials or a detector globally.
|
|
108
|
+
|
|
109
|
+
Run Bandit against both packages' `src`, `tools` and `examples` directories with
|
|
110
|
+
all rules, `--ignore-nosec` and JSON output. Preserve nonzero exits and collection
|
|
111
|
+
errors. Review SQL construction, subprocess arguments and inherited environment,
|
|
112
|
+
assertions and parser boundaries individually. A name-based warning can confuse
|
|
113
|
+
EquiFrame's local `load_dataset` with another project's API; prove the imported
|
|
114
|
+
call target before dismissing it. A subprocess argument list prevents shell
|
|
115
|
+
interpolation but still requires a trusted executable and trusted operations.
|
|
116
|
+
Do not pass publication credentials to build, test or package-execution jobs.
|
|
117
|
+
|
|
118
|
+
`tools/security_requirements.py` exports every public name/version pair from the
|
|
119
|
+
two locks, including alternative Python/platform pins. It puts alternative
|
|
120
|
+
versions of one project in separate files because pip-audit rejects duplicate
|
|
121
|
+
names in one input. From the repository root, export to a new temporary directory:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
python tools/security_requirements.py \
|
|
125
|
+
--lock uv.lock \
|
|
126
|
+
--lock packages/equiframe-benchmark/uv.lock \
|
|
127
|
+
--output /absolute/new/temporary/audit-requirements
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
For **each** exported requirements file, run the pinned `pip-audit` with
|
|
131
|
+
`--no-deps --disable-pip --strict --format json --progress-spinner off -r FILE`.
|
|
132
|
+
This sends public dependency names and versions to the selected advisory service;
|
|
133
|
+
it neither installs the project nor uploads its source. Preserve exits, all
|
|
134
|
+
advisories and skipped/failed collections. Also review admitted lower bounds and
|
|
135
|
+
new build dependencies: a clean resolved lock does not certify every version
|
|
136
|
+
allowed by a range. Do not use `--fix`, ignore lists or a zero-exit option to
|
|
137
|
+
manufacture a passing release.
|
|
138
|
+
|
|
139
|
+
Check exact distribution names and normalized spelling against the official
|
|
140
|
+
project documentation and PyPI metadata. Use one reviewed index, pinned hashes
|
|
141
|
+
and the exact expected package names. Similar spelling, a public metadata page or
|
|
142
|
+
a 404 does not authenticate a publisher, reserve a name or prove that a package
|
|
143
|
+
is benign. Never publish placeholder projects to reserve lookalike names.
|
|
144
|
+
|
|
145
|
+
The current release task uses manual publication, without GitHub Actions or
|
|
146
|
+
Trusted Publisher configuration. Use a locally handled publication token only
|
|
147
|
+
after the exact release gates pass. Limit its lifetime by revoking it after use.
|
|
148
|
+
Never put it in a command
|
|
149
|
+
argument, repository, persistent environment file, workflow secret or receipt;
|
|
150
|
+
use a hidden interactive prompt or an approved local secret store. Revoke it
|
|
151
|
+
after the authorized upload. Build and inspect without publication credentials,
|
|
152
|
+
then upload only the four approved files to the explicit index endpoint. TestPyPI
|
|
153
|
+
and PyPI have separate accounts. Reconcile every accepted filename/hash and test
|
|
154
|
+
the downloaded artifacts before promoting identical bytes to production. A
|
|
155
|
+
publication credential does not establish software safety or human acceptance.
|
|
156
|
+
|
|
157
|
+
Primary references: [TruffleHog usage](https://github.com/trufflesecurity/trufflehog),
|
|
158
|
+
[Bandit CLI](https://bandit.readthedocs.io/en/latest/man/bandit.html),
|
|
159
|
+
[pip-audit](https://github.com/pypa/pip-audit),
|
|
160
|
+
[PyPI account security](https://docs.pypi.org/your-account/two-factor-authentication/)
|
|
161
|
+
and [PyPA name normalization](https://packaging.python.org/en/latest/specifications/name-normalization/).
|
|
162
|
+
|
|
163
|
+
## Consolidation review boundary, 2026-10-06
|
|
164
|
+
|
|
165
|
+
The following paragraphs describe the historical 2026-10-06 consolidation.
|
|
166
|
+
That consolidation did **not** resume the stopped source-security assessment,
|
|
167
|
+
through a specialized scanner or through an equivalent manual audit. That is a
|
|
168
|
+
real unresolved coverage boundary, distinct from a temporarily missing tool.
|
|
169
|
+
The prior special-file observation and capture-budget contract question remain
|
|
170
|
+
open; no new reproduction, severity confirmation or remediation is claimed for
|
|
171
|
+
them. Ordinary functional tests, compatibility checks and documented resource
|
|
172
|
+
measurements do not close the security assessment.
|
|
173
|
+
|
|
174
|
+
A separate fresh reading of the official advisory for **CVE-2026-25087** on
|
|
175
|
+
2026-10-06 confirms Apache's stated scope: Arrow C++ 15.0.0 through 23.0.0 is
|
|
176
|
+
affected and 23.0.1 contains the fix; Python, Ruby and C GLib bindings are excluded
|
|
177
|
+
because the affected IPC pre-buffering functionality is not exposed through them.
|
|
178
|
+
This qualifies the historical minimum-profile package-version match. It does not
|
|
179
|
+
turn an inventory into a clean scan or prove every bundled native component and
|
|
180
|
+
EquiFrame usage safe. Sources: [Apache advisory](https://security.apache.org/projects/arrow/)
|
|
181
|
+
and [Arrow 23.0.1 release notes](https://arrow.apache.org/blog/2026/02/16/23.0.1-release/).
|
|
182
|
+
The dated source receipt is retained with this consolidation's evidence, separately
|
|
183
|
+
from earlier advisory snapshots. No dependency version was changed on that basis.
|
|
184
|
+
|
|
185
|
+
The final artifact-bound offline dependency inventories/SBOMs identify the new
|
|
186
|
+
wheel/sdist bytes. Offline regeneration performs no new registry-wide advisory
|
|
187
|
+
query and must not be presented as a current comprehensive vulnerability scan.
|
|
188
|
+
At that consolidation, release and untrusted-input adoption remained blocked on
|
|
189
|
+
the unresolved assessment and owner decisions. Current release decisions follow
|
|
190
|
+
the release gates; this historical snapshot is not a security certification.
|
|
191
|
+
|
|
192
|
+
## A14 secure-development follow-up, 2026-10-13
|
|
193
|
+
|
|
194
|
+
Stored operational failure summaries and CLI diagnostics use fixed messages.
|
|
195
|
+
They do not stringify external exceptions or publish their input values, source
|
|
196
|
+
paths or SQL. Optional DEBUG diagnostics contain at most 32 owned frame records
|
|
197
|
+
(module, function, line); external frames are anonymous and values, source text
|
|
198
|
+
and local variables are excluded. A broken logging handler cannot replace the
|
|
199
|
+
operational result. Programmatic validation exceptions and explicitly requested
|
|
200
|
+
raw data remain separate; applications must handle them before showing a UI.
|
|
201
|
+
|
|
202
|
+
EDA cancellation callbacks must be callable and return actual booleans. A failing
|
|
203
|
+
callback is not called again during error recovery; incomplete analysis persists
|
|
204
|
+
a closed diagnostic code. Unexpected optional-chart failures also remain
|
|
205
|
+
explicitly not evaluable, without exposing third-party exception text.
|
|
206
|
+
|
|
207
|
+
Evidence readers and bundle inventories inspect lexical ancestors before
|
|
208
|
+
resolution and reject symlinks, Windows reparse points and non-regular files.
|
|
209
|
+
POSIX reads pin directory descriptors and use O_NOFOLLOW/O_NONBLOCK; fallback
|
|
210
|
+
hosts check identities around opening. JSON and bundle documents retain their
|
|
211
|
+
64 MiB limit, and the bundle pin is checked against the bytes actually parsed.
|
|
212
|
+
These cooperative checks are not a filesystem sandbox against hostile mounts
|
|
213
|
+
or concurrent owners. Bundle copies still require a controlled, immutable source.
|
|
214
|
+
|
|
215
|
+
Contract and generic profile/detector document files also use protected regular
|
|
216
|
+
reads with a 64 MiB limit. Generic document readers retain OS exception classes
|
|
217
|
+
and errno for recovery, with fixed messages and no filenames. Relative output
|
|
218
|
+
directories retain relative member names in the published inventory.
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
# Semantics, guarantees and limits
|
|
2
|
+
|
|
3
|
+
[Handbook](index.md) · [API](api.md) · [Security](security.md)
|
|
4
|
+
|
|
5
|
+
## What equivalence means
|
|
6
|
+
|
|
7
|
+
`ComparisonResult` evaluates the contract's **output schema** over two captured
|
|
8
|
+
results. `equivalent` means a complete scan found no differences or output-contract
|
|
9
|
+
violations under the declared policies. It does not prove arbitrary programs
|
|
10
|
+
equivalent, validate an entire business process, evaluate operational R rules,
|
|
11
|
+
establish an independent expected output, or authorize data publication.
|
|
12
|
+
|
|
13
|
+
`diff` treats the right side as `expected` by default. `compare` defaults to
|
|
14
|
+
`peer`; explicitly pass `reference="expected"` when the right side is your
|
|
15
|
+
reference. An invalid expected result yields `not_evaluable`, including when a
|
|
16
|
+
scan completed. Two peers can agree on the same incorrect output.
|
|
17
|
+
|
|
18
|
+
| State | Scan/diagnostic interpretation |
|
|
19
|
+
|---|---|
|
|
20
|
+
| `equivalent` | Complete evaluation; no differences or violations in scope |
|
|
21
|
+
| `different` | Complete evaluation; differences or violations retained |
|
|
22
|
+
| `not_evaluable` | Required judgment unavailable; partial findings may remain |
|
|
23
|
+
|
|
24
|
+
Inspect `state`, `complete`, diagnostics and `counts.final` separately. A complete
|
|
25
|
+
scan is necessary but not sufficient for equivalence. Counts from interrupted
|
|
26
|
+
scans are partial observations. `difference_records` counts diagnostic records,
|
|
27
|
+
not necessarily distinct affected rows. Missing files, failed finalization,
|
|
28
|
+
resource exhaustion and exceptions never imply a pass.
|
|
29
|
+
|
|
30
|
+
Native [windows](migration-ir.md#deterministic-windows) preserve every input row.
|
|
31
|
+
Lag/lead retain Decimal precision/scale and units, integer width, timestamp
|
|
32
|
+
unit/zone, and the separate NULL/NaN meaning of the input. They introduce NULL
|
|
33
|
+
only at partition boundaries; input NULL values also remain NULL. Row-number
|
|
34
|
+
output is signed64 and nonnullable. Partition/order ties fail rather than select
|
|
35
|
+
an arbitrary original position. Explicit final sorting remains necessary for a
|
|
36
|
+
sequence contract. The compiler rejects incompatible declared types/units and
|
|
37
|
+
unsupported native plans before execution; comparison does not repair them.
|
|
38
|
+
|
|
39
|
+
Use `assert_equivalent`, which requires complete/final `observed_contract`
|
|
40
|
+
equivalence and any additional policy/evidence requirements. `ComparisonResult`
|
|
41
|
+
and `EquivalenceDecision` reject truth testing; use the assertion or the explicit
|
|
42
|
+
`.accepted` field. Presentation reports cannot be passed to this gate.
|
|
43
|
+
|
|
44
|
+
## Explicit contract, conservative inference
|
|
45
|
+
|
|
46
|
+
The observed physical schema is captured separately from the expected logical
|
|
47
|
+
schema. EquiFrame does not silently cast to the expected schema, fill NULL, trim
|
|
48
|
+
text, normalize Unicode, deduplicate, reorder or round values to remove a
|
|
49
|
+
mismatch. Any allowed equivalence must be declared, justified where required,
|
|
50
|
+
and visible in the recorded policy.
|
|
51
|
+
|
|
52
|
+
Typed contract models are strict and frozen, reject unknown fields and use
|
|
53
|
+
explicit tagged scalars. Contract validation checks metadata and cross-field
|
|
54
|
+
relationships; it does not evaluate actual tables or execute declared rules.
|
|
55
|
+
A valid JSON Schema document alone is not proof of all cross-field constraints.
|
|
56
|
+
Use the public validator and the appropriate data/evidence API.
|
|
57
|
+
|
|
58
|
+
Without an explicit contract, `diff` infers only from the right-hand Arrow
|
|
59
|
+
schema after faithful capture. It records provenance, defaults to nullable,
|
|
60
|
+
retains NULL separately from NaN and uses exact numbers with unit `unspecified`.
|
|
61
|
+
It does not learn tolerance, business keys, money units, non-nullability or
|
|
62
|
+
business meaning from samples. `ColumnSpec` overrides are the route to explicit
|
|
63
|
+
domain requirements.
|
|
64
|
+
|
|
65
|
+
## Schema, types and metadata
|
|
66
|
+
|
|
67
|
+
Default `ComparisonPolicy` is `column_order="significant"`,
|
|
68
|
+
`physical_types="exact"`, `index="reject"`. Nonconservative permissions require a
|
|
69
|
+
justification. Column names, authorized order, nullability, allowed values,
|
|
70
|
+
ranges and granularities are evaluated, even if both inputs violate them in the
|
|
71
|
+
same way. Empty tables retain and validate their schema.
|
|
72
|
+
|
|
73
|
+
"Exact physical types" uses the comparator's documented logical signature:
|
|
74
|
+
booleans; integer width/sign; float width; decimal precision/scale; strings;
|
|
75
|
+
dates; timestamp unit/timezone. Arrow string, large_string and string_view
|
|
76
|
+
represent the same string family; decimal128/256 of identical precision/scale
|
|
77
|
+
represent the same decimal; date32/date64 represent days, with whole-day checks.
|
|
78
|
+
This is not native dtype identity or arbitrary metadata equality.
|
|
79
|
+
|
|
80
|
+
`physical_types="lossless"` permits checked representable variations **within**
|
|
81
|
+
a family: integer width/sign, exactly representable float narrowing and exact
|
|
82
|
+
Decimal rescaling/range. It does not equate integer, float, Decimal and string.
|
|
83
|
+
An Arrow null-typed column needs explicit lossless permission and a nullable
|
|
84
|
+
contract to specialize. The expected domain is not invented.
|
|
85
|
+
|
|
86
|
+
Direct `diff` capture rejects uninterpreted Arrow field/schema metadata,
|
|
87
|
+
meaningful unsupported pandas metadata, categories, heterogeneous object cells,
|
|
88
|
+
unsupported subclasses and undeclared dtype metadata. Rejection is preferable
|
|
89
|
+
to asserting an equality that erased a distinction. Existing artifact routes
|
|
90
|
+
retain their own documented admission; wrapping an input does not expand it.
|
|
91
|
+
|
|
92
|
+
## NULL, NaN, zero and text
|
|
93
|
+
|
|
94
|
+
NULL, NaN, empty text and numeric zero remain distinct unless `NullPolicy` or
|
|
95
|
+
`NaNPolicy` explicitly authorizes an equivalence. Such permissions do not add
|
|
96
|
+
missing-value distinctions absent from the original representation.
|
|
97
|
+
|
|
98
|
+
- NaN `forbid` is a violation; `distinct` is a category distinct from NULL and
|
|
99
|
+
numbers; `as_null` needs the declared nullable/equivalence policy
|
|
100
|
+
- NaN payload/signaling distinctions are not compared
|
|
101
|
+
- Positive and negative float zero have equal numeric value; evidence retains
|
|
102
|
+
the observed sign. This is not bitwise floating-point identity
|
|
103
|
+
- Text remains exact: no trimming, case folding, Unicode normalization,
|
|
104
|
+
accent/collation inference or removal of leading identifier zeroes
|
|
105
|
+
- A NumPy float column cannot recover None versus NaN already collapsed before
|
|
106
|
+
EquiFrame sees it. Separate Arrow masks or suitable admitted representations
|
|
107
|
+
can preserve distinctions that still exist
|
|
108
|
+
|
|
109
|
+
## Decimal, units and approximation
|
|
110
|
+
|
|
111
|
+
Decimal comparison uses exact rational arithmetic, never an intermediate float
|
|
112
|
+
or the consumer's mutable Decimal context. Declared precision/scale, units and
|
|
113
|
+
engine capabilities matter. A rounding policy describes the intended
|
|
114
|
+
transformation; it does not authorize repairing a result during comparison.
|
|
115
|
+
Currency conversion is not inferred. A declared unit absent from the actual
|
|
116
|
+
data cannot be empirically recovered from a number alone.
|
|
117
|
+
|
|
118
|
+
For integer/float measures with an explicitly fixed tolerance:
|
|
119
|
+
|
|
120
|
+
```text
|
|
121
|
+
abs(a - b) <= max(absolute or 0, (relative or 0) * max(abs(a), abs(b)))
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The formula is symmetric and inclusive. Finite IEEE values and decimal-textual
|
|
125
|
+
tolerances are evaluated exactly as rationals. Infinities match only the same
|
|
126
|
+
sign, and NaN follows its separate policy. Decimal remains exact. Tolerances are
|
|
127
|
+
not estimated from the pair and do not define keys, uniqueness or ordering.
|
|
128
|
+
Approximate equality is not transitive.
|
|
129
|
+
|
|
130
|
+
Exact multisets count complete canonical values. Approximate multiset matching
|
|
131
|
+
uses bounded maximum matching to preserve occurrences; it is not a greedy
|
|
132
|
+
nearest-row heuristic. Work, candidate-pair and matching-memory limits can make
|
|
133
|
+
a case `not_evaluable`. Hashes partition work or verify bytes; hash equality
|
|
134
|
+
alone never substitutes for exact value comparison.
|
|
135
|
+
|
|
136
|
+
## Keys, duplicates and order
|
|
137
|
+
|
|
138
|
+
All declared keys are checked before and after authorized normalization. A
|
|
139
|
+
columns-scoped key includes its scope columns; table/global key scope covers the
|
|
140
|
+
compared output, not all external tables or future snapshots.
|
|
141
|
+
|
|
142
|
+
When row order is ignored, the first declared unique key is eligible for
|
|
143
|
+
alignment only after uniqueness and completeness are verified on both sides.
|
|
144
|
+
Otherwise the complete multiset is compared and key violations remain visible.
|
|
145
|
+
There are no approximate keys. `incomplete="allow"` retains incomplete rows and
|
|
146
|
+
excludes them from uniqueness checking; it does not fabricate identity or drop
|
|
147
|
+
those rows. Duplicate keys cannot be "fixed" by selecting the first occurrence.
|
|
148
|
+
|
|
149
|
+
`multiplicity="preserve"` counts every occurrence. `"distinct"` is a requirement
|
|
150
|
+
that normalized rows do not repeat, not permission to deduplicate. Without
|
|
151
|
+
verified identity, a report describes excess occurrences instead of inventing
|
|
152
|
+
modified business rows.
|
|
153
|
+
|
|
154
|
+
`SignificantOrder` compares sequence and validates the declared sort columns,
|
|
155
|
+
direction, NULL placement and ties. `ties="preserve"` retains observed relative
|
|
156
|
+
order; `ties="reject"` disallows ties. No sort repairs an output during comparison.
|
|
157
|
+
A pandas index is not automatically a key. A trivial unnamed
|
|
158
|
+
`RangeIndex(0, n, 1)` is not business identity; meaningful indices need promotion
|
|
159
|
+
to explicit columns or an explicit justified exclusion policy.
|
|
160
|
+
|
|
161
|
+
## Time and capture fidelity
|
|
162
|
+
|
|
163
|
+
Dates are whole days. Timestamps retain integer epochs and physical precision,
|
|
164
|
+
including nanoseconds, without a float or Python datetime rounding step.
|
|
165
|
+
`instant` and `wall_time` are separate policies. Changing timezone requires the
|
|
166
|
+
appropriate lossless/instant policy; local time requires the declared zone.
|
|
167
|
+
Naive timestamps can be preserved, rejected or interpreted using the declared
|
|
168
|
+
zone and DST policy. Nonexistent local DST times are violations; calendar/zone
|
|
169
|
+
operations outside the supported year range are not evaluated.
|
|
170
|
+
|
|
171
|
+
Contract timezone logic uses Python `tzdata`. General native transformations
|
|
172
|
+
use the provider available to their engine: pandas' input provider, Polars'
|
|
173
|
+
compiled provider, or DuckDB's ICU. Compiler-generated IR `LocalizeTimestamp`
|
|
174
|
+
in Polars instead reads the installed TZif metadata and evaluates its transition
|
|
175
|
+
rules with native Polars expressions. Resource hashes participate in the
|
|
176
|
+
compilation identity, and a resource changed before execution is rejected.
|
|
177
|
+
EquiFrame does not assume all providers share an IANA version. Engine, runtime,
|
|
178
|
+
resource and configuration facts remain explicit; tested agreement has a finite
|
|
179
|
+
scope and does not establish identical timezone databases.
|
|
180
|
+
|
|
181
|
+
Arrow does not retain each row's original ISO text/offset, negative Decimal zero
|
|
182
|
+
or distinct Decimal exponents within a fixed-scale column. Store those required
|
|
183
|
+
facts as explicit columns **before** they are lost. EquiFrame does not reconstruct
|
|
184
|
+
them after conversion.
|
|
185
|
+
|
|
186
|
+
## Capture, execution and migration are different capabilities
|
|
187
|
+
|
|
188
|
+
| Route | Admitted scope and important rejection |
|
|
189
|
+
|---|---|
|
|
190
|
+
| Direct `diff` | Eager pandas/Polars DataFrames, Arrow tables and explicitly supplied DuckDBPyRelation objects; relation submission authorizes bounded materialization; no inferred SQL strings, connections, Polars lazy plans or consumable readers |
|
|
191
|
+
| `compare` | `ArrowInput`, local `ArtifactRef`, local Arrow IPC file/Parquet `Path`, `TableArtifact`, completed `ExecutionResult` |
|
|
192
|
+
| Native pandas | Admitted scalar bool, integers, text, floats, decimals and temporal types; unsupported metadata/object/category cases rejected |
|
|
193
|
+
| Native Polars | Decimal precision at most 38 and nonnegative scale; date32; seconds promoted to milliseconds only when representable |
|
|
194
|
+
| Native DuckDB | Faithful admitted native types; decimals at most precision/scale 38; zoned timestamps limited to representable microseconds |
|
|
195
|
+
| Declarative migration | Bounded typed relational IR over each engine's genuine native implementation |
|
|
196
|
+
|
|
197
|
+
Native transport excludes dictionaries/categories, nested tabular values, binary,
|
|
198
|
+
arbitrary objects and Arrow streams. Capture support does not imply that every
|
|
199
|
+
native transformation supports every admitted type. Consult
|
|
200
|
+
`get_adapter_capabilities()` for stage-specific conditions and known losses.
|
|
201
|
+
|
|
202
|
+
DuckDB HUGEINT/UHUGEINT, UUID/ENUM and other types without declared faithful
|
|
203
|
+
mapping are rejected before Arrow conversion; temporal infinities and zoned
|
|
204
|
+
nanosecond timestamps are also rejected. Decimal SQL parameters are bound with
|
|
205
|
+
exact checked precision/scale. Nonfinite Decimal, negative zero, positive
|
|
206
|
+
exponent, excessive precision/scale and integers outside signed/unsigned 64-bit
|
|
207
|
+
range are rejected. A consumer-chosen explicit cast requires a justified
|
|
208
|
+
representable domain; EquiFrame never inserts it to make results match.
|
|
209
|
+
|
|
210
|
+
The migration IR includes Scan, Project, Filter, exact Cast, EquiJoin,
|
|
211
|
+
Aggregate, Sort, Concatenate and Deduplicate, with closed typed expressions
|
|
212
|
+
including Arithmetic, Coalesce and Conditional. It has explicit NULL-join,
|
|
213
|
+
NULL-group, empty-sum, tie and multiplicity policies. It does not translate
|
|
214
|
+
arbitrary Python/SQL. Its common decimal precision is at most 38. Right/full joins, matchable-key
|
|
215
|
+
cardinality, deterministic key deduplication, exact Decimal multiplication,
|
|
216
|
+
declared rounding/division and timestamp transport/localization have the
|
|
217
|
+
[per-engine contracts and bounds](migration-ir.md). Float join/group/order
|
|
218
|
+
keys and float SUM/min/max remain outside this IR.
|
|
219
|
+
`count_valid` can admit float. Native admission can reject a mathematically safe
|
|
220
|
+
case conservatively, for example a Polars sum with cancelling large terms.
|
|
221
|
+
|
|
222
|
+
Arithmetic `overflow="reject"` applies to every intermediate expression, even
|
|
223
|
+
when a later expression would cancel its value or produce a boolean or NULL.
|
|
224
|
+
Pandas and Polars evaluate those checks eagerly. DuckDB compiler v8 and its
|
|
225
|
+
native adapter consume each fallible expression and relational node at an
|
|
226
|
+
in-memory native checkpoint before a containing predicate or later operation
|
|
227
|
+
can prune it. The [checkpoint contract](migration-ir.md#eager-failures-and-historical-evidence)
|
|
228
|
+
defines adapter admission, resource checks and the remaining common IR limits.
|
|
229
|
+
No fallback or lossy cast supplies an answer.
|
|
230
|
+
|
|
231
|
+
`check_migration` checks both compilers without reading inputs or loading engines.
|
|
232
|
+
`migrate` captures the same inputs once and executes two native pipelines. The
|
|
233
|
+
optional expected output is separate and never feeds the transformations,
|
|
234
|
+
profiles or detectors. `require_rules=False` is an explicit change of requested
|
|
235
|
+
scope, not proof that R passed; child pipeline states and partial alerts remain.
|
|
236
|
+
|
|
237
|
+
## Evidence, privacy and execution limits
|
|
238
|
+
|
|
239
|
+
- Capture and comparison retain snapshots and full differences subject to budgets.
|
|
240
|
+
A report example limit does not reduce evaluation coverage
|
|
241
|
+
- Default comparison reports hide values, names and paths. Optional disclosure
|
|
242
|
+
must be deliberate. Redacting a view does not erase the raw evidence directory
|
|
243
|
+
- `read_comparison` validates the manifest; `iter_differences` verifies the
|
|
244
|
+
difference file's size/hash while reading it. Other readers have their own
|
|
245
|
+
closed-inventory and semantic checks; they are not interchangeable
|
|
246
|
+
- Report `diagnostic`, `detail` and `reproduction` modes offer different evidence.
|
|
247
|
+
Reproduction exports supported materialized inputs, not original transformations
|
|
248
|
+
or external services. `require_exhaustive` is independent of shown example count
|
|
249
|
+
- Hashes identify bytes relative to a trusted reference. They do not authenticate
|
|
250
|
+
authors/reviewers, prove expected-output correctness or certify a dependency safe
|
|
251
|
+
- Local files require a suitable filesystem, including exclusive atomic hard-link
|
|
252
|
+
publication where used. Publication failures can raise and leave partial files
|
|
253
|
+
- No API guarantees a report after process kill, disk failure or host OOM. CI must
|
|
254
|
+
fail closed on missing finalization or abnormal termination
|
|
255
|
+
- Core time, memory, row, disk and work checks are cooperative. Native allocations
|
|
256
|
+
may precede checkpoints. DuckDB engine limits are not total-process RSS quotas
|
|
257
|
+
- Callbacks, registered plugins, SQL and model-related integrations must be trusted.
|
|
258
|
+
EquiFrame is not a sandbox or a service concurrency/security controller
|
|
259
|
+
|
|
260
|
+
See [performance](performance.md), [security](security.md) and
|
|
261
|
+
[upgrading](upgrading.md) for operating and recovery procedures.
|
|
262
|
+
|
|
263
|
+
## Rules and anomaly signals are not equivalence
|
|
264
|
+
|
|
265
|
+
R evaluates declared rules over one execution; M is Isolation Forest; B is MAD;
|
|
266
|
+
H is R OR M; O is independent expected-output/review evidence. They remain
|
|
267
|
+
separate. Profiles and detectors do not receive labels, mutations or another
|
|
268
|
+
engine's output. Fitting, calibration and inference are distinct operations.
|
|
269
|
+
|
|
270
|
+
An unavailable component is `not_evaluable`, never `no_alert`. H retains an alert
|
|
271
|
+
from either R or M even if the other component is unavailable, while marking
|
|
272
|
+
coverage incomplete. With no alert and an unavailable component, H is not
|
|
273
|
+
evaluable. B does not participate in H. Empty rules do not establish R coverage.
|
|
274
|
+
A low anomaly score or `no_alert` is not an equivalence gate or load authorization.
|