equiframe 0.12.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- equiframe/__init__.py +832 -0
- equiframe/__main__.py +3 -0
- equiframe/_base.py +84 -0
- equiframe/_cli_support.py +125 -0
- equiframe/_comparison_io.py +161 -0
- equiframe/_dataset_provenance.py +146 -0
- equiframe/_decimal.py +22 -0
- equiframe/_diff_capture.py +433 -0
- equiframe/_doctor.py +123 -0
- equiframe/_document_io.py +61 -0
- equiframe/_documents.py +79 -0
- equiframe/_duckdb_transport.py +149 -0
- equiframe/_error_reporting.py +52 -0
- equiframe/_first_use.py +102 -0
- equiframe/_path_compat.py +26 -0
- equiframe/_physical.py +133 -0
- equiframe/_r_review.R +308 -0
- equiframe/_regular_io.py +167 -0
- equiframe/_report_format_v1.py +298 -0
- equiframe/_report_io.py +591 -0
- equiframe/_stability_inventory.py +1421 -0
- equiframe/_timezones.py +257 -0
- equiframe/_vendor/cyclonedx/LICENSE +201 -0
- equiframe/_vendor/cyclonedx/PROVENANCE.md +18 -0
- equiframe/_vendor/cyclonedx/bom-1.6.schema.json +5673 -0
- equiframe/_vendor/cyclonedx/jsf-0.82.schema.json +240 -0
- equiframe/_vendor/cyclonedx/spdx.schema.json +712 -0
- equiframe/adapters.py +454 -0
- equiframe/assertion_types.py +122 -0
- equiframe/assertions.py +271 -0
- equiframe/benchmark.py +101 -0
- equiframe/bundle.py +240 -0
- equiframe/cli.py +693 -0
- equiframe/comparison.py +937 -0
- equiframe/comparison_report.py +660 -0
- equiframe/comparison_types.py +257 -0
- equiframe/compatibility.py +546 -0
- equiframe/contracts.py +879 -0
- equiframe/datasets.py +1301 -0
- equiframe/decision.py +34 -0
- equiframe/destinations.py +295 -0
- equiframe/detector_types.py +431 -0
- equiframe/detectors.py +389 -0
- equiframe/diff.py +192 -0
- equiframe/diff_contracts.py +523 -0
- equiframe/docs/adoption.md +329 -0
- equiframe/docs/api.md +534 -0
- equiframe/docs/arquitecto.md +143 -0
- equiframe/docs/choosing-api.md +84 -0
- equiframe/docs/cientifico.md +180 -0
- equiframe/docs/compatibility.md +254 -0
- equiframe/docs/contributing.md +171 -0
- equiframe/docs/dependencies.md +237 -0
- equiframe/docs/eda.md +166 -0
- equiframe/docs/extending.md +158 -0
- equiframe/docs/fast-ci.md +52 -0
- equiframe/docs/first-use.md +78 -0
- equiframe/docs/index.md +142 -0
- equiframe/docs/ingestion.md +145 -0
- equiframe/docs/inicio.md +129 -0
- equiframe/docs/installed-private.md +97 -0
- equiframe/docs/journeys.md +362 -0
- equiframe/docs/lectura-conversion.md +211 -0
- equiframe/docs/licensing.md +17 -0
- equiframe/docs/maintenance.md +233 -0
- equiframe/docs/migration-ir.md +333 -0
- equiframe/docs/option-audit.md +45 -0
- equiframe/docs/outcome-gates.md +134 -0
- equiframe/docs/performance.md +263 -0
- equiframe/docs/piloto-simulado.md +56 -0
- equiframe/docs/quickstart.md +156 -0
- equiframe/docs/r_review.md +212 -0
- equiframe/docs/recipes.md +248 -0
- equiframe/docs/releasing.md +183 -0
- equiframe/docs/security.md +218 -0
- equiframe/docs/semantics.md +274 -0
- equiframe/docs/sqlite-snapshots.md +262 -0
- equiframe/docs/universal-sources.md +49 -0
- equiframe/docs/universalidad-cinco-pasos.md +107 -0
- equiframe/docs/upgrading.md +538 -0
- equiframe/duckdb_capture.py +410 -0
- equiframe/eda/__init__.py +36 -0
- equiframe/eda/__main__.py +3 -0
- equiframe/eda/_backend.py +612 -0
- equiframe/eda/_inventory.py +49 -0
- equiframe/eda/_paths.py +49 -0
- equiframe/eda/_report.py +291 -0
- equiframe/eda/api.py +535 -0
- equiframe/eda/cli.py +102 -0
- equiframe/eda/models.py +239 -0
- equiframe/errors.py +83 -0
- equiframe/evidence.py +375 -0
- equiframe/examples/integrations/eda_generic.py +60 -0
- equiframe/examples/integrations/outcome_gates.py +94 -0
- equiframe/examples/integrations/sqlite_snapshot.py +246 -0
- equiframe/examples/integrations/sqlite_snapshot_demo.py +226 -0
- equiframe/examples/integrations/universal_first_use.py +296 -0
- equiframe/examples/journeys/README.md +31 -0
- equiframe/examples/journeys/__init__.py +1 -0
- equiframe/examples/journeys/a_pandas_refactor.py +88 -0
- equiframe/examples/journeys/b_native_migration.py +181 -0
- equiframe/examples/journeys/c_retail_etl.py +460 -0
- equiframe/examples/journeys/common.py +136 -0
- equiframe/examples/journeys/d_financial_reconciliation.py +96 -0
- equiframe/examples/journeys/e_telemetry.py +135 -0
- equiframe/examples/journeys/f_data_science_preparation.py +84 -0
- equiframe/examples/journeys/g_external_snapshots.py +197 -0
- equiframe/examples/journeys/h_portable_ci.py +129 -0
- equiframe/examples/journeys/requirements.lock +249 -0
- equiframe/examples/journeys/verify_journeys.py +117 -0
- equiframe/examples/load_convert_compare.py +85 -0
- equiframe/execution.py +355 -0
- equiframe/execution_types.py +177 -0
- equiframe/ingestion.py +744 -0
- equiframe/local.py +126 -0
- equiframe/migration/__init__.py +94 -0
- equiframe/migration/_physical.py +8 -0
- equiframe/migration/backends/_timestamp_transport.py +109 -0
- equiframe/migration/backends/duckdb.py +651 -0
- equiframe/migration/backends/duckdb_decimal.py +125 -0
- equiframe/migration/backends/duckdb_time.py +87 -0
- equiframe/migration/backends/pandas.py +544 -0
- equiframe/migration/backends/polars.py +915 -0
- equiframe/migration/backends/polars_decimal.py +231 -0
- equiframe/migration/io.py +198 -0
- equiframe/migration/ports.py +96 -0
- equiframe/migration/runner.py +545 -0
- equiframe/migration/runner_types.py +405 -0
- equiframe/migration/types.py +354 -0
- equiframe/migration/validation.py +619 -0
- equiframe/normalization.py +358 -0
- equiframe/pipeline.py +274 -0
- equiframe/pipeline_types.py +247 -0
- equiframe/ports.py +39 -0
- equiframe/preprocessing.py +110 -0
- equiframe/profile_types.py +343 -0
- equiframe/profiles.py +447 -0
- equiframe/py.typed +0 -0
- equiframe/r_review.py +706 -0
- equiframe/reduction.py +518 -0
- equiframe/registry.py +121 -0
- equiframe/report_rendering.py +366 -0
- equiframe/report_types.py +294 -0
- equiframe/reporting.py +26 -0
- equiframe/rule_types.py +178 -0
- equiframe/rules.py +819 -0
- equiframe/schemas/r-execution-binding-1.0.schema.json +46 -0
- equiframe/schemas/r-review-binding-1.0.schema.json +28 -0
- equiframe/schemas/r-review-options-1.0.schema.json +38 -0
- equiframe/schemas/r-review-result-1.0.schema.json +207 -0
- equiframe/schemas/reviewed-decision-1.0.schema.json +3394 -0
- equiframe/serialization.py +152 -0
- equiframe/snapshot.py +296 -0
- equiframe/sources.py +283 -0
- equiframe/tables.py +459 -0
- equiframe/types.py +71 -0
- equiframe/values.py +252 -0
- equiframe/verification_request.py +35 -0
- equiframe/verify.py +799 -0
- equiframe/verify_types.py +255 -0
- equiframe/workflow.py +510 -0
- equiframe/workflow_request.py +38 -0
- equiframe-0.12.4.dist-info/METADATA +268 -0
- equiframe-0.12.4.dist-info/RECORD +170 -0
- equiframe-0.12.4.dist-info/WHEEL +4 -0
- equiframe-0.12.4.dist-info/entry_points.txt +3 -0
- equiframe-0.12.4.dist-info/licenses/LICENSE +21 -0
- equiframe-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md +42 -0
- equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/LICENSE +201 -0
- equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/PROVENANCE.md +18 -0
equiframe/reduction.py
ADDED
|
@@ -0,0 +1,518 @@
|
|
|
1
|
+
"""Deterministic, budgeted row deletion preserving an explicitly observed failure."""
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from collections.abc import Callable, Mapping
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from types import MappingProxyType
|
|
7
|
+
from typing import Annotated, Literal
|
|
8
|
+
from uuid import uuid4
|
|
9
|
+
|
|
10
|
+
from pydantic import BeforeValidator, Field, model_validator
|
|
11
|
+
from typing_extensions import Self
|
|
12
|
+
|
|
13
|
+
from equiframe._base import (
|
|
14
|
+
FrozenModel,
|
|
15
|
+
Identifier,
|
|
16
|
+
NonNegativeInt,
|
|
17
|
+
PositiveInt,
|
|
18
|
+
Text,
|
|
19
|
+
_ExplicitOutcome,
|
|
20
|
+
strict_bool_literal,
|
|
21
|
+
)
|
|
22
|
+
from equiframe._regular_io import local_output_directory
|
|
23
|
+
from equiframe.evidence import (
|
|
24
|
+
MAX_DOCUMENT_BYTES,
|
|
25
|
+
EventJournal,
|
|
26
|
+
EvidenceEvent,
|
|
27
|
+
inventory,
|
|
28
|
+
load_json,
|
|
29
|
+
parse_json,
|
|
30
|
+
publish_json,
|
|
31
|
+
verify_events,
|
|
32
|
+
verify_files,
|
|
33
|
+
)
|
|
34
|
+
from equiframe.execution import CancellationToken
|
|
35
|
+
from equiframe.local import LocalArtifactStore, local_path
|
|
36
|
+
from equiframe.pipeline_types import EvidenceFile
|
|
37
|
+
from equiframe.tables import ArrowInput
|
|
38
|
+
from equiframe.types import ArtifactRef
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ReductionOptions(FrozenModel):
|
|
42
|
+
max_evaluations: PositiveInt = 100
|
|
43
|
+
timeout_seconds: Annotated[float, Field(gt=0)] = 60.0
|
|
44
|
+
confirmations: Annotated[int, Field(ge=1, le=5)] = 2
|
|
45
|
+
max_input_rows: PositiveInt = 100_000
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class ReductionObservation(FrozenModel):
|
|
49
|
+
preconditions: Literal["satisfied", "violated", "not_evaluable"]
|
|
50
|
+
failure_signature: Identifier | None = None
|
|
51
|
+
|
|
52
|
+
@model_validator(mode="after")
|
|
53
|
+
def consistent(self) -> Self:
|
|
54
|
+
if self.preconditions != "satisfied" and self.failure_signature is not None:
|
|
55
|
+
raise ValueError("a failure is admissible only with satisfied preconditions")
|
|
56
|
+
return self
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class ReducedTable(FrozenModel):
|
|
60
|
+
name: Identifier
|
|
61
|
+
original: ArtifactRef
|
|
62
|
+
reduced: ArtifactRef
|
|
63
|
+
original_rows: NonNegativeInt
|
|
64
|
+
retained_positions: tuple[NonNegativeInt, ...]
|
|
65
|
+
|
|
66
|
+
@model_validator(mode="after")
|
|
67
|
+
def consistent(self) -> Self:
|
|
68
|
+
if tuple(sorted(set(self.retained_positions))) != self.retained_positions or any(
|
|
69
|
+
index >= self.original_rows for index in self.retained_positions
|
|
70
|
+
):
|
|
71
|
+
raise ValueError("retained positions must be unique, ordered original row positions")
|
|
72
|
+
return self
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class ReductionResult(_ExplicitOutcome, FrozenModel):
|
|
76
|
+
_truth_value_guidance = "inspect .status, .baseline_verified and .minimality explicitly"
|
|
77
|
+
|
|
78
|
+
schema_version: Literal["1.0"] = "1.0"
|
|
79
|
+
reduction_id: Identifier
|
|
80
|
+
predicate_version: Text
|
|
81
|
+
failure_signature: Identifier
|
|
82
|
+
options: ReductionOptions
|
|
83
|
+
status: Literal[
|
|
84
|
+
"reduced",
|
|
85
|
+
"unchanged",
|
|
86
|
+
"budget_exhausted",
|
|
87
|
+
"cancelled",
|
|
88
|
+
"invalid_baseline",
|
|
89
|
+
"evaluation_failed",
|
|
90
|
+
]
|
|
91
|
+
baseline_verified: bool
|
|
92
|
+
evaluations: NonNegativeInt
|
|
93
|
+
accepted_candidates: NonNegativeInt
|
|
94
|
+
elapsed_seconds: Annotated[float, Field(ge=0)]
|
|
95
|
+
minimality: Literal["one_minimal", "not_established"]
|
|
96
|
+
global_minimum_claimed: Annotated[Literal[False], BeforeValidator(strict_bool_literal)] = False
|
|
97
|
+
tables: tuple[ReducedTable, ...]
|
|
98
|
+
evidence: tuple[EvidenceFile, ...]
|
|
99
|
+
|
|
100
|
+
@model_validator(mode="after")
|
|
101
|
+
def consistent(self) -> Self:
|
|
102
|
+
if self.evaluations > self.options.max_evaluations:
|
|
103
|
+
raise ValueError("evaluation budget exceeded")
|
|
104
|
+
if self.minimality == "one_minimal" and (
|
|
105
|
+
not self.baseline_verified or self.status not in ("reduced", "unchanged")
|
|
106
|
+
):
|
|
107
|
+
raise ValueError("minimality requires a verified, exhaustive local search")
|
|
108
|
+
if not self.baseline_verified and self.accepted_candidates:
|
|
109
|
+
raise ValueError("cannot accept reductions of an unverified baseline")
|
|
110
|
+
if len({table.name for table in self.tables}) != len(self.tables):
|
|
111
|
+
raise ValueError("duplicate table names")
|
|
112
|
+
if len({item.path for item in self.evidence}) != len(self.evidence):
|
|
113
|
+
raise ValueError("duplicate evidence paths")
|
|
114
|
+
return self
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class _Stop(Exception):
|
|
118
|
+
def __init__(self, status: Literal["budget_exhausted", "cancelled", "evaluation_failed"]):
|
|
119
|
+
self.status = status
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def reduce_case(
|
|
123
|
+
inputs: Mapping[str, ArrowInput],
|
|
124
|
+
*,
|
|
125
|
+
predicate: Callable[[Mapping[str, ArrowInput]], ReductionObservation],
|
|
126
|
+
predicate_version: str,
|
|
127
|
+
failure_signature: str,
|
|
128
|
+
options: ReductionOptions | None = None,
|
|
129
|
+
work_dir: Path | None = None,
|
|
130
|
+
cancellation: CancellationToken | None = None,
|
|
131
|
+
) -> ReductionResult:
|
|
132
|
+
"""Delete row chunks across tables, preserving order, schema and duplicate identity.
|
|
133
|
+
|
|
134
|
+
The trusted predicate must check every required precondition and reproduce the
|
|
135
|
+
SAME target failure. No SQL/callback is loaded from JSON. Timings are cooperative;
|
|
136
|
+
a blocking callback cannot be preempted. A successful late callback is not accepted.
|
|
137
|
+
"""
|
|
138
|
+
import pyarrow as pa
|
|
139
|
+
from pydantic import TypeAdapter
|
|
140
|
+
|
|
141
|
+
options = ReductionOptions.model_validate(ReductionOptions() if options is None else options)
|
|
142
|
+
TypeAdapter(Identifier).validate_python(failure_signature, strict=True)
|
|
143
|
+
TypeAdapter(Text).validate_python(predicate_version, strict=True)
|
|
144
|
+
names = tuple(sorted(inputs))
|
|
145
|
+
for name in names:
|
|
146
|
+
TypeAdapter(Identifier).validate_python(name, strict=True)
|
|
147
|
+
if not names or any(not isinstance(inputs[name], ArrowInput) for name in names):
|
|
148
|
+
raise ValueError("named Arrow inputs are required")
|
|
149
|
+
tables = {name: inputs[name].table for name in names}
|
|
150
|
+
if any(not isinstance(table, pa.Table) for table in tables.values()):
|
|
151
|
+
raise ValueError("reduction requires materialized Arrow tables")
|
|
152
|
+
if sum(table.num_rows for table in tables.values()) > options.max_input_rows:
|
|
153
|
+
raise ValueError("reduction input row budget exceeded")
|
|
154
|
+
identifier = "reduction_" + uuid4().hex
|
|
155
|
+
directory = (
|
|
156
|
+
local_output_directory(Path.cwd() / "equiframe-runs" if work_dir is None else work_dir)
|
|
157
|
+
/ identifier
|
|
158
|
+
)
|
|
159
|
+
directory.mkdir(parents=True, exist_ok=False)
|
|
160
|
+
token = cancellation if cancellation is not None else CancellationToken()
|
|
161
|
+
journal = EventJournal(directory, identifier)
|
|
162
|
+
journal.emit("reduction", "started", "reduction_started")
|
|
163
|
+
publish_json(options, directory / "options.json")
|
|
164
|
+
store = LocalArtifactStore(directory / "artifacts")
|
|
165
|
+
|
|
166
|
+
def capture(name: str, table: pa.Table, kind: str) -> ArtifactRef:
|
|
167
|
+
path = directory / f"{name}.{kind}.arrow"
|
|
168
|
+
with path.open("xb") as stream, pa.ipc.new_file(stream, table.schema) as writer:
|
|
169
|
+
writer.write_table(table)
|
|
170
|
+
return store.put(path, media_type="application/vnd.apache.arrow.file")
|
|
171
|
+
|
|
172
|
+
original = {name: capture(name, tables[name], "original") for name in names}
|
|
173
|
+
selected = tuple((name, index) for name in names for index in range(tables[name].num_rows))
|
|
174
|
+
evaluations = accepted = 0
|
|
175
|
+
start = time.perf_counter()
|
|
176
|
+
deadline = start + options.timeout_seconds
|
|
177
|
+
|
|
178
|
+
def check_budget(*, count: bool) -> None:
|
|
179
|
+
if token.cancelled:
|
|
180
|
+
raise _Stop("cancelled")
|
|
181
|
+
if time.perf_counter() >= deadline or (count and evaluations >= options.max_evaluations):
|
|
182
|
+
raise _Stop("budget_exhausted")
|
|
183
|
+
|
|
184
|
+
def positions(candidate: tuple[tuple[str, int], ...]) -> dict[str, tuple[int, ...]]:
|
|
185
|
+
grouped: dict[str, list[int]] = {name: [] for name in names}
|
|
186
|
+
for name, index in candidate:
|
|
187
|
+
grouped[name].append(index)
|
|
188
|
+
return {name: tuple(grouped[name]) for name in names}
|
|
189
|
+
|
|
190
|
+
def materialize(candidate: tuple[tuple[str, int], ...]) -> Mapping[str, ArrowInput]:
|
|
191
|
+
indexes = positions(candidate)
|
|
192
|
+
return MappingProxyType(
|
|
193
|
+
{
|
|
194
|
+
name: ArrowInput(tables[name].take(pa.array(indexes[name], type=pa.int64())))
|
|
195
|
+
for name in names
|
|
196
|
+
}
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
def preserves(candidate: tuple[tuple[str, int], ...]) -> bool:
|
|
200
|
+
nonlocal evaluations
|
|
201
|
+
for _ in range(options.confirmations):
|
|
202
|
+
check_budget(count=True)
|
|
203
|
+
evaluations += 1
|
|
204
|
+
attempt = directory / f"attempt-{evaluations:06d}"
|
|
205
|
+
attempt.mkdir()
|
|
206
|
+
publish_json({"positions": positions(candidate)}, attempt / "selection.json")
|
|
207
|
+
journal.emit("predicate", "started", "predicate_started")
|
|
208
|
+
try:
|
|
209
|
+
observation = ReductionObservation.model_validate(predicate(materialize(candidate)))
|
|
210
|
+
except Exception:
|
|
211
|
+
publish_json({"code": "predicate_failed"}, attempt / "error.json")
|
|
212
|
+
journal.emit("predicate", "failed", "predicate_failed")
|
|
213
|
+
raise _Stop("evaluation_failed") from None
|
|
214
|
+
publish_json(observation, attempt / "observation.json")
|
|
215
|
+
matched = observation.preconditions == "satisfied" and (
|
|
216
|
+
observation.failure_signature == failure_signature
|
|
217
|
+
)
|
|
218
|
+
journal.emit("predicate", "finished", "matched" if matched else "not_preserved")
|
|
219
|
+
check_budget(count=False)
|
|
220
|
+
if not matched:
|
|
221
|
+
return False
|
|
222
|
+
return True
|
|
223
|
+
|
|
224
|
+
baseline_verified = False
|
|
225
|
+
status: Literal[
|
|
226
|
+
"reduced",
|
|
227
|
+
"unchanged",
|
|
228
|
+
"budget_exhausted",
|
|
229
|
+
"cancelled",
|
|
230
|
+
"invalid_baseline",
|
|
231
|
+
"evaluation_failed",
|
|
232
|
+
] = "unchanged"
|
|
233
|
+
minimality: Literal["one_minimal", "not_established"] = "not_established"
|
|
234
|
+
try:
|
|
235
|
+
baseline_verified = preserves(selected)
|
|
236
|
+
if not baseline_verified:
|
|
237
|
+
status = "invalid_baseline"
|
|
238
|
+
else:
|
|
239
|
+
granularity = 2
|
|
240
|
+
while selected:
|
|
241
|
+
width = max(1, (len(selected) + granularity - 1) // granularity)
|
|
242
|
+
changed = False
|
|
243
|
+
for offset in range(0, len(selected), width):
|
|
244
|
+
candidate = selected[:offset] + selected[offset + width :]
|
|
245
|
+
if preserves(candidate):
|
|
246
|
+
selected = candidate
|
|
247
|
+
accepted += 1
|
|
248
|
+
journal.emit("reduction", "accepted", "failure_preserved")
|
|
249
|
+
granularity = max(2, granularity - 1)
|
|
250
|
+
changed = True
|
|
251
|
+
break
|
|
252
|
+
if changed:
|
|
253
|
+
continue
|
|
254
|
+
if width == 1:
|
|
255
|
+
break
|
|
256
|
+
granularity = min(len(selected), granularity * 2)
|
|
257
|
+
status = "reduced" if accepted else "unchanged"
|
|
258
|
+
minimality = "one_minimal"
|
|
259
|
+
except _Stop as stopped:
|
|
260
|
+
status = stopped.status
|
|
261
|
+
retained = positions(selected)
|
|
262
|
+
reduced = materialize(selected)
|
|
263
|
+
results = tuple(
|
|
264
|
+
ReducedTable(
|
|
265
|
+
name=name,
|
|
266
|
+
original=original[name],
|
|
267
|
+
reduced=capture(name, reduced[name].table, "reduced"),
|
|
268
|
+
original_rows=tables[name].num_rows,
|
|
269
|
+
retained_positions=retained[name],
|
|
270
|
+
)
|
|
271
|
+
for name in names
|
|
272
|
+
)
|
|
273
|
+
journal.emit("reduction", "finished", status)
|
|
274
|
+
result = ReductionResult(
|
|
275
|
+
reduction_id=identifier,
|
|
276
|
+
predicate_version=predicate_version,
|
|
277
|
+
failure_signature=failure_signature,
|
|
278
|
+
options=options,
|
|
279
|
+
status=status,
|
|
280
|
+
baseline_verified=baseline_verified,
|
|
281
|
+
evaluations=evaluations,
|
|
282
|
+
accepted_candidates=accepted,
|
|
283
|
+
elapsed_seconds=time.perf_counter() - start,
|
|
284
|
+
minimality=minimality,
|
|
285
|
+
tables=results,
|
|
286
|
+
evidence=inventory(directory),
|
|
287
|
+
)
|
|
288
|
+
publish_json(result, directory / "reduction.json")
|
|
289
|
+
return result
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def read_reduction(path: Path) -> ReductionResult:
|
|
293
|
+
"""Verify recorded decisions and row provenance without invoking the predicate."""
|
|
294
|
+
path = path / "reduction.json" if path.is_dir() else path
|
|
295
|
+
result = load_json(ReductionResult, path)
|
|
296
|
+
directory = path.parent
|
|
297
|
+
verify_files(directory, result.evidence)
|
|
298
|
+
from equiframe.errors import ArtifactIntegrityError
|
|
299
|
+
|
|
300
|
+
listed = {item.path: item for item in result.evidence}
|
|
301
|
+
if not {"events.jsonl", "options.json"} <= listed.keys() or not result.tables:
|
|
302
|
+
raise ArtifactIntegrityError("reduction is missing required evidence")
|
|
303
|
+
if load_json(ReductionOptions, directory / "options.json") != result.options:
|
|
304
|
+
raise ArtifactIntegrityError("reduction options contradict the recorded configuration")
|
|
305
|
+
events = verify_events(directory / "events.jsonl", operation_id=result.reduction_id)
|
|
306
|
+
if (
|
|
307
|
+
len(events) < 2
|
|
308
|
+
or (events[0].stage, events[0].state, events[0].code)
|
|
309
|
+
!= ("reduction", "started", "reduction_started")
|
|
310
|
+
or events[-1].stage != "reduction"
|
|
311
|
+
or events[-1].state != "finished"
|
|
312
|
+
or events[-1].code != result.status
|
|
313
|
+
or sum(e.stage == "predicate" and e.state == "started" for e in events)
|
|
314
|
+
!= result.evaluations
|
|
315
|
+
or sum(e.stage == "reduction" and e.state == "accepted" for e in events)
|
|
316
|
+
!= result.accepted_candidates
|
|
317
|
+
):
|
|
318
|
+
raise ArtifactIntegrityError("reduction journal contradicts final result")
|
|
319
|
+
_verify_reduction_tables(result, directory, listed)
|
|
320
|
+
_verify_reduction_attempts(result, directory, listed, events)
|
|
321
|
+
return result
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _verify_reduction_tables(
|
|
325
|
+
result: ReductionResult, directory: Path, listed: Mapping[str, EvidenceFile]
|
|
326
|
+
) -> None:
|
|
327
|
+
import hashlib
|
|
328
|
+
|
|
329
|
+
import pyarrow as pa
|
|
330
|
+
|
|
331
|
+
from equiframe.errors import ArtifactIntegrityError
|
|
332
|
+
|
|
333
|
+
root = directory.resolve()
|
|
334
|
+
total_rows = 0
|
|
335
|
+
for table in result.tables:
|
|
336
|
+
paths: dict[str, Path] = {}
|
|
337
|
+
for kind, ref in (("original", table.original), ("reduced", table.reduced)):
|
|
338
|
+
named = listed.get(f"{table.name}.{kind}.arrow")
|
|
339
|
+
artifact = local_path(ref).resolve()
|
|
340
|
+
if not artifact.is_relative_to(root):
|
|
341
|
+
raise ArtifactIntegrityError("reduction artifact escapes its operation directory")
|
|
342
|
+
item = listed.get(artifact.relative_to(root).as_posix())
|
|
343
|
+
if (
|
|
344
|
+
named is None
|
|
345
|
+
or item is None
|
|
346
|
+
or ref.format != "arrow"
|
|
347
|
+
or ref.media_type != "application/vnd.apache.arrow.file"
|
|
348
|
+
or ref.schema_ref is not None
|
|
349
|
+
or (ref.sha256, ref.size_bytes) != (item.sha256, item.size_bytes)
|
|
350
|
+
or (ref.sha256, ref.size_bytes) != (named.sha256, named.size_bytes)
|
|
351
|
+
):
|
|
352
|
+
raise ArtifactIntegrityError("reduction artifact contradicts its captured table")
|
|
353
|
+
paths[kind] = artifact
|
|
354
|
+
with paths["original"].open("rb") as stream, pa.ipc.open_file(stream) as reader:
|
|
355
|
+
original = reader.read_all()
|
|
356
|
+
original.validate(full=True)
|
|
357
|
+
total_rows += original.num_rows
|
|
358
|
+
if original.num_rows != table.original_rows or total_rows > result.options.max_input_rows:
|
|
359
|
+
raise ArtifactIntegrityError("original table row count contradicts the reduction")
|
|
360
|
+
expected = original.take(pa.array(table.retained_positions, type=pa.int64()))
|
|
361
|
+
sink = pa.BufferOutputStream()
|
|
362
|
+
with pa.ipc.new_file(sink, expected.schema) as writer:
|
|
363
|
+
writer.write_table(expected)
|
|
364
|
+
if hashlib.sha256(sink.getvalue()).hexdigest() != table.reduced.sha256:
|
|
365
|
+
raise ArtifactIntegrityError(
|
|
366
|
+
"reduced table does not preserve its recorded row positions"
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _verify_reduction_attempts(
|
|
371
|
+
result: ReductionResult,
|
|
372
|
+
directory: Path,
|
|
373
|
+
listed: Mapping[str, EvidenceFile],
|
|
374
|
+
events: tuple[EvidenceEvent, ...],
|
|
375
|
+
) -> None:
|
|
376
|
+
from pydantic import TypeAdapter
|
|
377
|
+
|
|
378
|
+
from equiframe.errors import ArtifactIntegrityError
|
|
379
|
+
|
|
380
|
+
names = tuple(table.name for table in result.tables)
|
|
381
|
+
original_rows = {table.name: table.original_rows for table in result.tables}
|
|
382
|
+
current = tuple((name, index) for name in names for index in range(original_rows[name]))
|
|
383
|
+
initial = current
|
|
384
|
+
pending: tuple[tuple[str, int], ...] | None = None
|
|
385
|
+
matches = evaluation = accepted = baseline_matches = 0
|
|
386
|
+
rejected: set[tuple[tuple[str, int], ...]] = set()
|
|
387
|
+
required_attempts: set[str] = set()
|
|
388
|
+
last_outcome = ""
|
|
389
|
+
index = 1
|
|
390
|
+
while index < len(events) - 1:
|
|
391
|
+
event = events[index]
|
|
392
|
+
if (event.stage, event.state, event.code) == ("reduction", "accepted", "failure_preserved"):
|
|
393
|
+
if pending is None or matches != result.options.confirmations:
|
|
394
|
+
raise ArtifactIntegrityError("accepted reduction lacks matching confirmations")
|
|
395
|
+
current = pending
|
|
396
|
+
accepted += 1
|
|
397
|
+
pending = None
|
|
398
|
+
matches = 0
|
|
399
|
+
rejected.clear()
|
|
400
|
+
index += 1
|
|
401
|
+
continue
|
|
402
|
+
if (event.stage, event.state, event.code) != ("predicate", "started", "predicate_started"):
|
|
403
|
+
raise ArtifactIntegrityError("unexpected reduction journal transition")
|
|
404
|
+
evaluation += 1
|
|
405
|
+
attempt = f"attempt-{evaluation:06d}/"
|
|
406
|
+
selection_file = attempt + "selection.json"
|
|
407
|
+
required_attempts.add(selection_file)
|
|
408
|
+
if selection_file not in listed:
|
|
409
|
+
raise ArtifactIntegrityError("predicate selection is missing from the inventory")
|
|
410
|
+
with (directory / selection_file).open("rb") as stream:
|
|
411
|
+
data = stream.read(MAX_DOCUMENT_BYTES + 1)
|
|
412
|
+
if len(data) > MAX_DOCUMENT_BYTES:
|
|
413
|
+
raise ArtifactIntegrityError("predicate selection exceeds the verification budget")
|
|
414
|
+
selection = parse_json(data)
|
|
415
|
+
if not isinstance(selection, dict) or set(selection) != {"positions"}:
|
|
416
|
+
raise ArtifactIntegrityError("invalid predicate selection")
|
|
417
|
+
positions = selection["positions"]
|
|
418
|
+
if not isinstance(positions, dict) or set(positions) != set(names):
|
|
419
|
+
raise ArtifactIntegrityError("predicate selection has different input tables")
|
|
420
|
+
candidate_items: list[tuple[str, int]] = []
|
|
421
|
+
for name in names:
|
|
422
|
+
indexes = positions[name]
|
|
423
|
+
if not isinstance(indexes, list):
|
|
424
|
+
raise ArtifactIntegrityError("invalid predicate row positions")
|
|
425
|
+
validated = tuple(
|
|
426
|
+
TypeAdapter(NonNegativeInt).validate_python(i, strict=True) for i in indexes
|
|
427
|
+
)
|
|
428
|
+
if tuple(sorted(set(validated))) != validated or any(
|
|
429
|
+
i >= original_rows[name] for i in validated
|
|
430
|
+
):
|
|
431
|
+
raise ArtifactIntegrityError("invalid predicate row positions")
|
|
432
|
+
candidate_items.extend((name, i) for i in validated)
|
|
433
|
+
candidate = tuple(candidate_items)
|
|
434
|
+
baseline = evaluation <= result.options.confirmations
|
|
435
|
+
if not result.baseline_verified or baseline:
|
|
436
|
+
if candidate != initial:
|
|
437
|
+
raise ArtifactIntegrityError("baseline observations belong to different rows")
|
|
438
|
+
elif not set(candidate) < set(current) or (pending is not None and candidate != pending):
|
|
439
|
+
raise ArtifactIntegrityError(
|
|
440
|
+
"predicate candidate is not a deletion of the current case"
|
|
441
|
+
)
|
|
442
|
+
index += 1
|
|
443
|
+
if index >= len(events) - 1:
|
|
444
|
+
raise ArtifactIntegrityError("predicate attempt has no recorded outcome")
|
|
445
|
+
outcome = events[index]
|
|
446
|
+
if (outcome.stage, outcome.state, outcome.code) == (
|
|
447
|
+
"predicate",
|
|
448
|
+
"failed",
|
|
449
|
+
"predicate_failed",
|
|
450
|
+
):
|
|
451
|
+
error_file = attempt + "error.json"
|
|
452
|
+
required_attempts.add(error_file)
|
|
453
|
+
if error_file not in listed or parse_json((directory / error_file).read_bytes()) != {
|
|
454
|
+
"code": "predicate_failed"
|
|
455
|
+
}:
|
|
456
|
+
raise ArtifactIntegrityError(
|
|
457
|
+
"predicate failure evidence is missing or inconsistent"
|
|
458
|
+
)
|
|
459
|
+
if result.status != "evaluation_failed" or index != len(events) - 2:
|
|
460
|
+
raise ArtifactIntegrityError("predicate failure contradicts the final status")
|
|
461
|
+
last_outcome = "predicate_failed"
|
|
462
|
+
index += 1
|
|
463
|
+
continue
|
|
464
|
+
observation_file = attempt + "observation.json"
|
|
465
|
+
required_attempts.add(observation_file)
|
|
466
|
+
if observation_file not in listed:
|
|
467
|
+
raise ArtifactIntegrityError("predicate observation is missing from the inventory")
|
|
468
|
+
observation = load_json(ReductionObservation, directory / observation_file)
|
|
469
|
+
matched = observation.preconditions == "satisfied" and (
|
|
470
|
+
observation.failure_signature == result.failure_signature
|
|
471
|
+
)
|
|
472
|
+
code = "matched" if matched else "not_preserved"
|
|
473
|
+
if (outcome.stage, outcome.state, outcome.code) != ("predicate", "finished", code):
|
|
474
|
+
raise ArtifactIntegrityError("predicate observation contradicts its journal outcome")
|
|
475
|
+
if result.baseline_verified and baseline:
|
|
476
|
+
if not matched:
|
|
477
|
+
raise ArtifactIntegrityError(
|
|
478
|
+
"verified baseline does not reproduce the target failure"
|
|
479
|
+
)
|
|
480
|
+
baseline_matches += 1
|
|
481
|
+
elif matched:
|
|
482
|
+
pending = candidate
|
|
483
|
+
matches += 1
|
|
484
|
+
if matches > result.options.confirmations:
|
|
485
|
+
raise ArtifactIntegrityError("predicate confirmations exceed the configured count")
|
|
486
|
+
else:
|
|
487
|
+
rejected.add(candidate)
|
|
488
|
+
pending = None
|
|
489
|
+
matches = 0
|
|
490
|
+
last_outcome = code
|
|
491
|
+
index += 1
|
|
492
|
+
if (
|
|
493
|
+
evaluation != result.evaluations
|
|
494
|
+
or accepted != result.accepted_candidates
|
|
495
|
+
or (result.baseline_verified and baseline_matches != result.options.confirmations)
|
|
496
|
+
or {item for item in listed if item.startswith("attempt-")} != required_attempts
|
|
497
|
+
or current
|
|
498
|
+
!= tuple((table.name, i) for table in result.tables for i in table.retained_positions)
|
|
499
|
+
):
|
|
500
|
+
raise ArtifactIntegrityError("reduction attempts contradict the final result")
|
|
501
|
+
if result.status == "invalid_baseline" and (
|
|
502
|
+
result.baseline_verified or last_outcome != "not_preserved"
|
|
503
|
+
):
|
|
504
|
+
raise ArtifactIntegrityError("invalid baseline status contradicts its observations")
|
|
505
|
+
if result.status == "evaluation_failed" and last_outcome != "predicate_failed":
|
|
506
|
+
raise ArtifactIntegrityError("evaluation failure lacks a failed predicate attempt")
|
|
507
|
+
if result.status in ("reduced", "unchanged"):
|
|
508
|
+
if (
|
|
509
|
+
not result.baseline_verified
|
|
510
|
+
or pending is not None
|
|
511
|
+
or ((result.status == "reduced") != bool(accepted))
|
|
512
|
+
):
|
|
513
|
+
raise ArtifactIntegrityError("successful reduction status contradicts its search")
|
|
514
|
+
if result.minimality == "one_minimal" and any(
|
|
515
|
+
current[:offset] + current[offset + 1 :] not in rejected
|
|
516
|
+
for offset in range(len(current))
|
|
517
|
+
):
|
|
518
|
+
raise ArtifactIntegrityError("one-minimal claim lacks the individual deletion checks")
|
equiframe/registry.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Explicit instance-owned registry of trusted native implementations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable, Mapping
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import TYPE_CHECKING, Literal, Protocol
|
|
8
|
+
|
|
9
|
+
from pydantic import ConfigDict, TypeAdapter
|
|
10
|
+
|
|
11
|
+
from equiframe._base import Identifier, PositiveInt, Text
|
|
12
|
+
from equiframe.errors import InvalidExecutionInput
|
|
13
|
+
from equiframe.execution_types import ExecutionPlan
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
import pandas as pd
|
|
17
|
+
import polars as pl
|
|
18
|
+
|
|
19
|
+
from equiframe.execution import RunContext
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Transformation(Protocol):
|
|
23
|
+
@property
|
|
24
|
+
def engine(self) -> str: ...
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def transformation_id(self) -> str: ...
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def revision(self) -> int: ...
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def code_version(self) -> str: ...
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _native_identity(
|
|
37
|
+
engine: str, expected: str, transformation_id: str, revision: int, code_version: str
|
|
38
|
+
) -> None:
|
|
39
|
+
"""Validate built-in metadata without importing or invoking native runtimes."""
|
|
40
|
+
safe_errors = ConfigDict(hide_input_in_errors=True)
|
|
41
|
+
TypeAdapter(Identifier, config=safe_errors).validate_python(transformation_id, strict=True)
|
|
42
|
+
TypeAdapter(PositiveInt, config=safe_errors).validate_python(revision, strict=True)
|
|
43
|
+
TypeAdapter(Text, config=safe_errors).validate_python(code_version, strict=True)
|
|
44
|
+
if type(engine) is not str or engine != expected:
|
|
45
|
+
raise InvalidExecutionInput("native transformation engine differs from its class")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class PandasTransformation:
|
|
50
|
+
transformation_id: str
|
|
51
|
+
callback: Callable[[Mapping[str, pd.DataFrame], RunContext], pd.DataFrame]
|
|
52
|
+
code_version: str
|
|
53
|
+
revision: int = 1
|
|
54
|
+
engine: Literal["pandas"] = "pandas"
|
|
55
|
+
|
|
56
|
+
def __post_init__(self) -> None:
|
|
57
|
+
_native_identity(
|
|
58
|
+
self.engine, "pandas", self.transformation_id, self.revision, self.code_version
|
|
59
|
+
)
|
|
60
|
+
if not callable(self.callback):
|
|
61
|
+
raise InvalidExecutionInput("native transformation requires a trusted callable")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class PolarsTransformation:
|
|
66
|
+
transformation_id: str
|
|
67
|
+
callback: Callable[[Mapping[str, pl.DataFrame], RunContext], pl.DataFrame | pl.LazyFrame]
|
|
68
|
+
code_version: str
|
|
69
|
+
revision: int = 1
|
|
70
|
+
engine: Literal["polars"] = "polars"
|
|
71
|
+
|
|
72
|
+
def __post_init__(self) -> None:
|
|
73
|
+
_native_identity(
|
|
74
|
+
self.engine, "polars", self.transformation_id, self.revision, self.code_version
|
|
75
|
+
)
|
|
76
|
+
if not callable(self.callback):
|
|
77
|
+
raise InvalidExecutionInput("native transformation requires a trusted callable")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class DuckDBTransformation:
|
|
82
|
+
transformation_id: str
|
|
83
|
+
sql: str
|
|
84
|
+
code_version: str
|
|
85
|
+
revision: int = 1
|
|
86
|
+
engine: Literal["duckdb"] = "duckdb"
|
|
87
|
+
|
|
88
|
+
def __post_init__(self) -> None:
|
|
89
|
+
_native_identity(
|
|
90
|
+
self.engine, "duckdb", self.transformation_id, self.revision, self.code_version
|
|
91
|
+
)
|
|
92
|
+
if not isinstance(self.sql, str) or not self.sql.strip() or "\x00" in self.sql:
|
|
93
|
+
raise InvalidExecutionInput("native transformation requires nonempty trusted SQL text")
|
|
94
|
+
if any(0xD800 <= ord(char) <= 0xDFFF for char in self.sql):
|
|
95
|
+
raise InvalidExecutionInput("native SQL requires valid Unicode scalar values")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class TransformationRegistry:
|
|
99
|
+
def __init__(self) -> None:
|
|
100
|
+
self._items: dict[tuple[str, str, int], Transformation] = {}
|
|
101
|
+
|
|
102
|
+
def register(self, transformation: Transformation) -> None:
|
|
103
|
+
from pydantic import TypeAdapter
|
|
104
|
+
|
|
105
|
+
TypeAdapter(Identifier).validate_python(transformation.engine, strict=True)
|
|
106
|
+
TypeAdapter(Identifier).validate_python(transformation.transformation_id, strict=True)
|
|
107
|
+
TypeAdapter(PositiveInt).validate_python(transformation.revision, strict=True)
|
|
108
|
+
TypeAdapter(Text).validate_python(transformation.code_version, strict=True)
|
|
109
|
+
key = (transformation.engine, transformation.transformation_id, transformation.revision)
|
|
110
|
+
if key in self._items:
|
|
111
|
+
raise InvalidExecutionInput("transformation already registered")
|
|
112
|
+
self._items[key] = transformation
|
|
113
|
+
|
|
114
|
+
def resolve(self, plan: ExecutionPlan) -> Transformation:
|
|
115
|
+
key = (plan.engine, plan.transformation_id, plan.transformation_revision)
|
|
116
|
+
try:
|
|
117
|
+
return self._items[key]
|
|
118
|
+
except KeyError:
|
|
119
|
+
raise InvalidExecutionInput(
|
|
120
|
+
"transformation is not registered for this engine/revision"
|
|
121
|
+
) from None
|