equiframe 0.12.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. equiframe/__init__.py +832 -0
  2. equiframe/__main__.py +3 -0
  3. equiframe/_base.py +84 -0
  4. equiframe/_cli_support.py +125 -0
  5. equiframe/_comparison_io.py +161 -0
  6. equiframe/_dataset_provenance.py +146 -0
  7. equiframe/_decimal.py +22 -0
  8. equiframe/_diff_capture.py +433 -0
  9. equiframe/_doctor.py +123 -0
  10. equiframe/_document_io.py +61 -0
  11. equiframe/_documents.py +79 -0
  12. equiframe/_duckdb_transport.py +149 -0
  13. equiframe/_error_reporting.py +52 -0
  14. equiframe/_first_use.py +102 -0
  15. equiframe/_path_compat.py +26 -0
  16. equiframe/_physical.py +133 -0
  17. equiframe/_r_review.R +308 -0
  18. equiframe/_regular_io.py +167 -0
  19. equiframe/_report_format_v1.py +298 -0
  20. equiframe/_report_io.py +591 -0
  21. equiframe/_stability_inventory.py +1421 -0
  22. equiframe/_timezones.py +257 -0
  23. equiframe/_vendor/cyclonedx/LICENSE +201 -0
  24. equiframe/_vendor/cyclonedx/PROVENANCE.md +18 -0
  25. equiframe/_vendor/cyclonedx/bom-1.6.schema.json +5673 -0
  26. equiframe/_vendor/cyclonedx/jsf-0.82.schema.json +240 -0
  27. equiframe/_vendor/cyclonedx/spdx.schema.json +712 -0
  28. equiframe/adapters.py +454 -0
  29. equiframe/assertion_types.py +122 -0
  30. equiframe/assertions.py +271 -0
  31. equiframe/benchmark.py +101 -0
  32. equiframe/bundle.py +240 -0
  33. equiframe/cli.py +693 -0
  34. equiframe/comparison.py +937 -0
  35. equiframe/comparison_report.py +660 -0
  36. equiframe/comparison_types.py +257 -0
  37. equiframe/compatibility.py +546 -0
  38. equiframe/contracts.py +879 -0
  39. equiframe/datasets.py +1301 -0
  40. equiframe/decision.py +34 -0
  41. equiframe/destinations.py +295 -0
  42. equiframe/detector_types.py +431 -0
  43. equiframe/detectors.py +389 -0
  44. equiframe/diff.py +192 -0
  45. equiframe/diff_contracts.py +523 -0
  46. equiframe/docs/adoption.md +329 -0
  47. equiframe/docs/api.md +534 -0
  48. equiframe/docs/arquitecto.md +143 -0
  49. equiframe/docs/choosing-api.md +84 -0
  50. equiframe/docs/cientifico.md +180 -0
  51. equiframe/docs/compatibility.md +254 -0
  52. equiframe/docs/contributing.md +171 -0
  53. equiframe/docs/dependencies.md +237 -0
  54. equiframe/docs/eda.md +166 -0
  55. equiframe/docs/extending.md +158 -0
  56. equiframe/docs/fast-ci.md +52 -0
  57. equiframe/docs/first-use.md +78 -0
  58. equiframe/docs/index.md +142 -0
  59. equiframe/docs/ingestion.md +145 -0
  60. equiframe/docs/inicio.md +129 -0
  61. equiframe/docs/installed-private.md +97 -0
  62. equiframe/docs/journeys.md +362 -0
  63. equiframe/docs/lectura-conversion.md +211 -0
  64. equiframe/docs/licensing.md +17 -0
  65. equiframe/docs/maintenance.md +233 -0
  66. equiframe/docs/migration-ir.md +333 -0
  67. equiframe/docs/option-audit.md +45 -0
  68. equiframe/docs/outcome-gates.md +134 -0
  69. equiframe/docs/performance.md +263 -0
  70. equiframe/docs/piloto-simulado.md +56 -0
  71. equiframe/docs/quickstart.md +156 -0
  72. equiframe/docs/r_review.md +212 -0
  73. equiframe/docs/recipes.md +248 -0
  74. equiframe/docs/releasing.md +183 -0
  75. equiframe/docs/security.md +218 -0
  76. equiframe/docs/semantics.md +274 -0
  77. equiframe/docs/sqlite-snapshots.md +262 -0
  78. equiframe/docs/universal-sources.md +49 -0
  79. equiframe/docs/universalidad-cinco-pasos.md +107 -0
  80. equiframe/docs/upgrading.md +538 -0
  81. equiframe/duckdb_capture.py +410 -0
  82. equiframe/eda/__init__.py +36 -0
  83. equiframe/eda/__main__.py +3 -0
  84. equiframe/eda/_backend.py +612 -0
  85. equiframe/eda/_inventory.py +49 -0
  86. equiframe/eda/_paths.py +49 -0
  87. equiframe/eda/_report.py +291 -0
  88. equiframe/eda/api.py +535 -0
  89. equiframe/eda/cli.py +102 -0
  90. equiframe/eda/models.py +239 -0
  91. equiframe/errors.py +83 -0
  92. equiframe/evidence.py +375 -0
  93. equiframe/examples/integrations/eda_generic.py +60 -0
  94. equiframe/examples/integrations/outcome_gates.py +94 -0
  95. equiframe/examples/integrations/sqlite_snapshot.py +246 -0
  96. equiframe/examples/integrations/sqlite_snapshot_demo.py +226 -0
  97. equiframe/examples/integrations/universal_first_use.py +296 -0
  98. equiframe/examples/journeys/README.md +31 -0
  99. equiframe/examples/journeys/__init__.py +1 -0
  100. equiframe/examples/journeys/a_pandas_refactor.py +88 -0
  101. equiframe/examples/journeys/b_native_migration.py +181 -0
  102. equiframe/examples/journeys/c_retail_etl.py +460 -0
  103. equiframe/examples/journeys/common.py +136 -0
  104. equiframe/examples/journeys/d_financial_reconciliation.py +96 -0
  105. equiframe/examples/journeys/e_telemetry.py +135 -0
  106. equiframe/examples/journeys/f_data_science_preparation.py +84 -0
  107. equiframe/examples/journeys/g_external_snapshots.py +197 -0
  108. equiframe/examples/journeys/h_portable_ci.py +129 -0
  109. equiframe/examples/journeys/requirements.lock +249 -0
  110. equiframe/examples/journeys/verify_journeys.py +117 -0
  111. equiframe/examples/load_convert_compare.py +85 -0
  112. equiframe/execution.py +355 -0
  113. equiframe/execution_types.py +177 -0
  114. equiframe/ingestion.py +744 -0
  115. equiframe/local.py +126 -0
  116. equiframe/migration/__init__.py +94 -0
  117. equiframe/migration/_physical.py +8 -0
  118. equiframe/migration/backends/_timestamp_transport.py +109 -0
  119. equiframe/migration/backends/duckdb.py +651 -0
  120. equiframe/migration/backends/duckdb_decimal.py +125 -0
  121. equiframe/migration/backends/duckdb_time.py +87 -0
  122. equiframe/migration/backends/pandas.py +544 -0
  123. equiframe/migration/backends/polars.py +915 -0
  124. equiframe/migration/backends/polars_decimal.py +231 -0
  125. equiframe/migration/io.py +198 -0
  126. equiframe/migration/ports.py +96 -0
  127. equiframe/migration/runner.py +545 -0
  128. equiframe/migration/runner_types.py +405 -0
  129. equiframe/migration/types.py +354 -0
  130. equiframe/migration/validation.py +619 -0
  131. equiframe/normalization.py +358 -0
  132. equiframe/pipeline.py +274 -0
  133. equiframe/pipeline_types.py +247 -0
  134. equiframe/ports.py +39 -0
  135. equiframe/preprocessing.py +110 -0
  136. equiframe/profile_types.py +343 -0
  137. equiframe/profiles.py +447 -0
  138. equiframe/py.typed +0 -0
  139. equiframe/r_review.py +706 -0
  140. equiframe/reduction.py +518 -0
  141. equiframe/registry.py +121 -0
  142. equiframe/report_rendering.py +366 -0
  143. equiframe/report_types.py +294 -0
  144. equiframe/reporting.py +26 -0
  145. equiframe/rule_types.py +178 -0
  146. equiframe/rules.py +819 -0
  147. equiframe/schemas/r-execution-binding-1.0.schema.json +46 -0
  148. equiframe/schemas/r-review-binding-1.0.schema.json +28 -0
  149. equiframe/schemas/r-review-options-1.0.schema.json +38 -0
  150. equiframe/schemas/r-review-result-1.0.schema.json +207 -0
  151. equiframe/schemas/reviewed-decision-1.0.schema.json +3394 -0
  152. equiframe/serialization.py +152 -0
  153. equiframe/snapshot.py +296 -0
  154. equiframe/sources.py +283 -0
  155. equiframe/tables.py +459 -0
  156. equiframe/types.py +71 -0
  157. equiframe/values.py +252 -0
  158. equiframe/verification_request.py +35 -0
  159. equiframe/verify.py +799 -0
  160. equiframe/verify_types.py +255 -0
  161. equiframe/workflow.py +510 -0
  162. equiframe/workflow_request.py +38 -0
  163. equiframe-0.12.4.dist-info/METADATA +268 -0
  164. equiframe-0.12.4.dist-info/RECORD +170 -0
  165. equiframe-0.12.4.dist-info/WHEEL +4 -0
  166. equiframe-0.12.4.dist-info/entry_points.txt +3 -0
  167. equiframe-0.12.4.dist-info/licenses/LICENSE +21 -0
  168. equiframe-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md +42 -0
  169. equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/LICENSE +201 -0
  170. equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/PROVENANCE.md +18 -0
equiframe/reduction.py ADDED
@@ -0,0 +1,518 @@
1
+ """Deterministic, budgeted row deletion preserving an explicitly observed failure."""
2
+
3
+ import time
4
+ from collections.abc import Callable, Mapping
5
+ from pathlib import Path
6
+ from types import MappingProxyType
7
+ from typing import Annotated, Literal
8
+ from uuid import uuid4
9
+
10
+ from pydantic import BeforeValidator, Field, model_validator
11
+ from typing_extensions import Self
12
+
13
+ from equiframe._base import (
14
+ FrozenModel,
15
+ Identifier,
16
+ NonNegativeInt,
17
+ PositiveInt,
18
+ Text,
19
+ _ExplicitOutcome,
20
+ strict_bool_literal,
21
+ )
22
+ from equiframe._regular_io import local_output_directory
23
+ from equiframe.evidence import (
24
+ MAX_DOCUMENT_BYTES,
25
+ EventJournal,
26
+ EvidenceEvent,
27
+ inventory,
28
+ load_json,
29
+ parse_json,
30
+ publish_json,
31
+ verify_events,
32
+ verify_files,
33
+ )
34
+ from equiframe.execution import CancellationToken
35
+ from equiframe.local import LocalArtifactStore, local_path
36
+ from equiframe.pipeline_types import EvidenceFile
37
+ from equiframe.tables import ArrowInput
38
+ from equiframe.types import ArtifactRef
39
+
40
+
41
+ class ReductionOptions(FrozenModel):
42
+ max_evaluations: PositiveInt = 100
43
+ timeout_seconds: Annotated[float, Field(gt=0)] = 60.0
44
+ confirmations: Annotated[int, Field(ge=1, le=5)] = 2
45
+ max_input_rows: PositiveInt = 100_000
46
+
47
+
48
+ class ReductionObservation(FrozenModel):
49
+ preconditions: Literal["satisfied", "violated", "not_evaluable"]
50
+ failure_signature: Identifier | None = None
51
+
52
+ @model_validator(mode="after")
53
+ def consistent(self) -> Self:
54
+ if self.preconditions != "satisfied" and self.failure_signature is not None:
55
+ raise ValueError("a failure is admissible only with satisfied preconditions")
56
+ return self
57
+
58
+
59
+ class ReducedTable(FrozenModel):
60
+ name: Identifier
61
+ original: ArtifactRef
62
+ reduced: ArtifactRef
63
+ original_rows: NonNegativeInt
64
+ retained_positions: tuple[NonNegativeInt, ...]
65
+
66
+ @model_validator(mode="after")
67
+ def consistent(self) -> Self:
68
+ if tuple(sorted(set(self.retained_positions))) != self.retained_positions or any(
69
+ index >= self.original_rows for index in self.retained_positions
70
+ ):
71
+ raise ValueError("retained positions must be unique, ordered original row positions")
72
+ return self
73
+
74
+
75
+ class ReductionResult(_ExplicitOutcome, FrozenModel):
76
+ _truth_value_guidance = "inspect .status, .baseline_verified and .minimality explicitly"
77
+
78
+ schema_version: Literal["1.0"] = "1.0"
79
+ reduction_id: Identifier
80
+ predicate_version: Text
81
+ failure_signature: Identifier
82
+ options: ReductionOptions
83
+ status: Literal[
84
+ "reduced",
85
+ "unchanged",
86
+ "budget_exhausted",
87
+ "cancelled",
88
+ "invalid_baseline",
89
+ "evaluation_failed",
90
+ ]
91
+ baseline_verified: bool
92
+ evaluations: NonNegativeInt
93
+ accepted_candidates: NonNegativeInt
94
+ elapsed_seconds: Annotated[float, Field(ge=0)]
95
+ minimality: Literal["one_minimal", "not_established"]
96
+ global_minimum_claimed: Annotated[Literal[False], BeforeValidator(strict_bool_literal)] = False
97
+ tables: tuple[ReducedTable, ...]
98
+ evidence: tuple[EvidenceFile, ...]
99
+
100
+ @model_validator(mode="after")
101
+ def consistent(self) -> Self:
102
+ if self.evaluations > self.options.max_evaluations:
103
+ raise ValueError("evaluation budget exceeded")
104
+ if self.minimality == "one_minimal" and (
105
+ not self.baseline_verified or self.status not in ("reduced", "unchanged")
106
+ ):
107
+ raise ValueError("minimality requires a verified, exhaustive local search")
108
+ if not self.baseline_verified and self.accepted_candidates:
109
+ raise ValueError("cannot accept reductions of an unverified baseline")
110
+ if len({table.name for table in self.tables}) != len(self.tables):
111
+ raise ValueError("duplicate table names")
112
+ if len({item.path for item in self.evidence}) != len(self.evidence):
113
+ raise ValueError("duplicate evidence paths")
114
+ return self
115
+
116
+
117
+ class _Stop(Exception):
118
+ def __init__(self, status: Literal["budget_exhausted", "cancelled", "evaluation_failed"]):
119
+ self.status = status
120
+
121
+
122
+ def reduce_case(
123
+ inputs: Mapping[str, ArrowInput],
124
+ *,
125
+ predicate: Callable[[Mapping[str, ArrowInput]], ReductionObservation],
126
+ predicate_version: str,
127
+ failure_signature: str,
128
+ options: ReductionOptions | None = None,
129
+ work_dir: Path | None = None,
130
+ cancellation: CancellationToken | None = None,
131
+ ) -> ReductionResult:
132
+ """Delete row chunks across tables, preserving order, schema and duplicate identity.
133
+
134
+ The trusted predicate must check every required precondition and reproduce the
135
+ SAME target failure. No SQL/callback is loaded from JSON. Timings are cooperative;
136
+ a blocking callback cannot be preempted. A successful late callback is not accepted.
137
+ """
138
+ import pyarrow as pa
139
+ from pydantic import TypeAdapter
140
+
141
+ options = ReductionOptions.model_validate(ReductionOptions() if options is None else options)
142
+ TypeAdapter(Identifier).validate_python(failure_signature, strict=True)
143
+ TypeAdapter(Text).validate_python(predicate_version, strict=True)
144
+ names = tuple(sorted(inputs))
145
+ for name in names:
146
+ TypeAdapter(Identifier).validate_python(name, strict=True)
147
+ if not names or any(not isinstance(inputs[name], ArrowInput) for name in names):
148
+ raise ValueError("named Arrow inputs are required")
149
+ tables = {name: inputs[name].table for name in names}
150
+ if any(not isinstance(table, pa.Table) for table in tables.values()):
151
+ raise ValueError("reduction requires materialized Arrow tables")
152
+ if sum(table.num_rows for table in tables.values()) > options.max_input_rows:
153
+ raise ValueError("reduction input row budget exceeded")
154
+ identifier = "reduction_" + uuid4().hex
155
+ directory = (
156
+ local_output_directory(Path.cwd() / "equiframe-runs" if work_dir is None else work_dir)
157
+ / identifier
158
+ )
159
+ directory.mkdir(parents=True, exist_ok=False)
160
+ token = cancellation if cancellation is not None else CancellationToken()
161
+ journal = EventJournal(directory, identifier)
162
+ journal.emit("reduction", "started", "reduction_started")
163
+ publish_json(options, directory / "options.json")
164
+ store = LocalArtifactStore(directory / "artifacts")
165
+
166
+ def capture(name: str, table: pa.Table, kind: str) -> ArtifactRef:
167
+ path = directory / f"{name}.{kind}.arrow"
168
+ with path.open("xb") as stream, pa.ipc.new_file(stream, table.schema) as writer:
169
+ writer.write_table(table)
170
+ return store.put(path, media_type="application/vnd.apache.arrow.file")
171
+
172
+ original = {name: capture(name, tables[name], "original") for name in names}
173
+ selected = tuple((name, index) for name in names for index in range(tables[name].num_rows))
174
+ evaluations = accepted = 0
175
+ start = time.perf_counter()
176
+ deadline = start + options.timeout_seconds
177
+
178
+ def check_budget(*, count: bool) -> None:
179
+ if token.cancelled:
180
+ raise _Stop("cancelled")
181
+ if time.perf_counter() >= deadline or (count and evaluations >= options.max_evaluations):
182
+ raise _Stop("budget_exhausted")
183
+
184
+ def positions(candidate: tuple[tuple[str, int], ...]) -> dict[str, tuple[int, ...]]:
185
+ grouped: dict[str, list[int]] = {name: [] for name in names}
186
+ for name, index in candidate:
187
+ grouped[name].append(index)
188
+ return {name: tuple(grouped[name]) for name in names}
189
+
190
+ def materialize(candidate: tuple[tuple[str, int], ...]) -> Mapping[str, ArrowInput]:
191
+ indexes = positions(candidate)
192
+ return MappingProxyType(
193
+ {
194
+ name: ArrowInput(tables[name].take(pa.array(indexes[name], type=pa.int64())))
195
+ for name in names
196
+ }
197
+ )
198
+
199
+ def preserves(candidate: tuple[tuple[str, int], ...]) -> bool:
200
+ nonlocal evaluations
201
+ for _ in range(options.confirmations):
202
+ check_budget(count=True)
203
+ evaluations += 1
204
+ attempt = directory / f"attempt-{evaluations:06d}"
205
+ attempt.mkdir()
206
+ publish_json({"positions": positions(candidate)}, attempt / "selection.json")
207
+ journal.emit("predicate", "started", "predicate_started")
208
+ try:
209
+ observation = ReductionObservation.model_validate(predicate(materialize(candidate)))
210
+ except Exception:
211
+ publish_json({"code": "predicate_failed"}, attempt / "error.json")
212
+ journal.emit("predicate", "failed", "predicate_failed")
213
+ raise _Stop("evaluation_failed") from None
214
+ publish_json(observation, attempt / "observation.json")
215
+ matched = observation.preconditions == "satisfied" and (
216
+ observation.failure_signature == failure_signature
217
+ )
218
+ journal.emit("predicate", "finished", "matched" if matched else "not_preserved")
219
+ check_budget(count=False)
220
+ if not matched:
221
+ return False
222
+ return True
223
+
224
+ baseline_verified = False
225
+ status: Literal[
226
+ "reduced",
227
+ "unchanged",
228
+ "budget_exhausted",
229
+ "cancelled",
230
+ "invalid_baseline",
231
+ "evaluation_failed",
232
+ ] = "unchanged"
233
+ minimality: Literal["one_minimal", "not_established"] = "not_established"
234
+ try:
235
+ baseline_verified = preserves(selected)
236
+ if not baseline_verified:
237
+ status = "invalid_baseline"
238
+ else:
239
+ granularity = 2
240
+ while selected:
241
+ width = max(1, (len(selected) + granularity - 1) // granularity)
242
+ changed = False
243
+ for offset in range(0, len(selected), width):
244
+ candidate = selected[:offset] + selected[offset + width :]
245
+ if preserves(candidate):
246
+ selected = candidate
247
+ accepted += 1
248
+ journal.emit("reduction", "accepted", "failure_preserved")
249
+ granularity = max(2, granularity - 1)
250
+ changed = True
251
+ break
252
+ if changed:
253
+ continue
254
+ if width == 1:
255
+ break
256
+ granularity = min(len(selected), granularity * 2)
257
+ status = "reduced" if accepted else "unchanged"
258
+ minimality = "one_minimal"
259
+ except _Stop as stopped:
260
+ status = stopped.status
261
+ retained = positions(selected)
262
+ reduced = materialize(selected)
263
+ results = tuple(
264
+ ReducedTable(
265
+ name=name,
266
+ original=original[name],
267
+ reduced=capture(name, reduced[name].table, "reduced"),
268
+ original_rows=tables[name].num_rows,
269
+ retained_positions=retained[name],
270
+ )
271
+ for name in names
272
+ )
273
+ journal.emit("reduction", "finished", status)
274
+ result = ReductionResult(
275
+ reduction_id=identifier,
276
+ predicate_version=predicate_version,
277
+ failure_signature=failure_signature,
278
+ options=options,
279
+ status=status,
280
+ baseline_verified=baseline_verified,
281
+ evaluations=evaluations,
282
+ accepted_candidates=accepted,
283
+ elapsed_seconds=time.perf_counter() - start,
284
+ minimality=minimality,
285
+ tables=results,
286
+ evidence=inventory(directory),
287
+ )
288
+ publish_json(result, directory / "reduction.json")
289
+ return result
290
+
291
+
292
+ def read_reduction(path: Path) -> ReductionResult:
293
+ """Verify recorded decisions and row provenance without invoking the predicate."""
294
+ path = path / "reduction.json" if path.is_dir() else path
295
+ result = load_json(ReductionResult, path)
296
+ directory = path.parent
297
+ verify_files(directory, result.evidence)
298
+ from equiframe.errors import ArtifactIntegrityError
299
+
300
+ listed = {item.path: item for item in result.evidence}
301
+ if not {"events.jsonl", "options.json"} <= listed.keys() or not result.tables:
302
+ raise ArtifactIntegrityError("reduction is missing required evidence")
303
+ if load_json(ReductionOptions, directory / "options.json") != result.options:
304
+ raise ArtifactIntegrityError("reduction options contradict the recorded configuration")
305
+ events = verify_events(directory / "events.jsonl", operation_id=result.reduction_id)
306
+ if (
307
+ len(events) < 2
308
+ or (events[0].stage, events[0].state, events[0].code)
309
+ != ("reduction", "started", "reduction_started")
310
+ or events[-1].stage != "reduction"
311
+ or events[-1].state != "finished"
312
+ or events[-1].code != result.status
313
+ or sum(e.stage == "predicate" and e.state == "started" for e in events)
314
+ != result.evaluations
315
+ or sum(e.stage == "reduction" and e.state == "accepted" for e in events)
316
+ != result.accepted_candidates
317
+ ):
318
+ raise ArtifactIntegrityError("reduction journal contradicts final result")
319
+ _verify_reduction_tables(result, directory, listed)
320
+ _verify_reduction_attempts(result, directory, listed, events)
321
+ return result
322
+
323
+
324
+ def _verify_reduction_tables(
325
+ result: ReductionResult, directory: Path, listed: Mapping[str, EvidenceFile]
326
+ ) -> None:
327
+ import hashlib
328
+
329
+ import pyarrow as pa
330
+
331
+ from equiframe.errors import ArtifactIntegrityError
332
+
333
+ root = directory.resolve()
334
+ total_rows = 0
335
+ for table in result.tables:
336
+ paths: dict[str, Path] = {}
337
+ for kind, ref in (("original", table.original), ("reduced", table.reduced)):
338
+ named = listed.get(f"{table.name}.{kind}.arrow")
339
+ artifact = local_path(ref).resolve()
340
+ if not artifact.is_relative_to(root):
341
+ raise ArtifactIntegrityError("reduction artifact escapes its operation directory")
342
+ item = listed.get(artifact.relative_to(root).as_posix())
343
+ if (
344
+ named is None
345
+ or item is None
346
+ or ref.format != "arrow"
347
+ or ref.media_type != "application/vnd.apache.arrow.file"
348
+ or ref.schema_ref is not None
349
+ or (ref.sha256, ref.size_bytes) != (item.sha256, item.size_bytes)
350
+ or (ref.sha256, ref.size_bytes) != (named.sha256, named.size_bytes)
351
+ ):
352
+ raise ArtifactIntegrityError("reduction artifact contradicts its captured table")
353
+ paths[kind] = artifact
354
+ with paths["original"].open("rb") as stream, pa.ipc.open_file(stream) as reader:
355
+ original = reader.read_all()
356
+ original.validate(full=True)
357
+ total_rows += original.num_rows
358
+ if original.num_rows != table.original_rows or total_rows > result.options.max_input_rows:
359
+ raise ArtifactIntegrityError("original table row count contradicts the reduction")
360
+ expected = original.take(pa.array(table.retained_positions, type=pa.int64()))
361
+ sink = pa.BufferOutputStream()
362
+ with pa.ipc.new_file(sink, expected.schema) as writer:
363
+ writer.write_table(expected)
364
+ if hashlib.sha256(sink.getvalue()).hexdigest() != table.reduced.sha256:
365
+ raise ArtifactIntegrityError(
366
+ "reduced table does not preserve its recorded row positions"
367
+ )
368
+
369
+
370
+ def _verify_reduction_attempts(
371
+ result: ReductionResult,
372
+ directory: Path,
373
+ listed: Mapping[str, EvidenceFile],
374
+ events: tuple[EvidenceEvent, ...],
375
+ ) -> None:
376
+ from pydantic import TypeAdapter
377
+
378
+ from equiframe.errors import ArtifactIntegrityError
379
+
380
+ names = tuple(table.name for table in result.tables)
381
+ original_rows = {table.name: table.original_rows for table in result.tables}
382
+ current = tuple((name, index) for name in names for index in range(original_rows[name]))
383
+ initial = current
384
+ pending: tuple[tuple[str, int], ...] | None = None
385
+ matches = evaluation = accepted = baseline_matches = 0
386
+ rejected: set[tuple[tuple[str, int], ...]] = set()
387
+ required_attempts: set[str] = set()
388
+ last_outcome = ""
389
+ index = 1
390
+ while index < len(events) - 1:
391
+ event = events[index]
392
+ if (event.stage, event.state, event.code) == ("reduction", "accepted", "failure_preserved"):
393
+ if pending is None or matches != result.options.confirmations:
394
+ raise ArtifactIntegrityError("accepted reduction lacks matching confirmations")
395
+ current = pending
396
+ accepted += 1
397
+ pending = None
398
+ matches = 0
399
+ rejected.clear()
400
+ index += 1
401
+ continue
402
+ if (event.stage, event.state, event.code) != ("predicate", "started", "predicate_started"):
403
+ raise ArtifactIntegrityError("unexpected reduction journal transition")
404
+ evaluation += 1
405
+ attempt = f"attempt-{evaluation:06d}/"
406
+ selection_file = attempt + "selection.json"
407
+ required_attempts.add(selection_file)
408
+ if selection_file not in listed:
409
+ raise ArtifactIntegrityError("predicate selection is missing from the inventory")
410
+ with (directory / selection_file).open("rb") as stream:
411
+ data = stream.read(MAX_DOCUMENT_BYTES + 1)
412
+ if len(data) > MAX_DOCUMENT_BYTES:
413
+ raise ArtifactIntegrityError("predicate selection exceeds the verification budget")
414
+ selection = parse_json(data)
415
+ if not isinstance(selection, dict) or set(selection) != {"positions"}:
416
+ raise ArtifactIntegrityError("invalid predicate selection")
417
+ positions = selection["positions"]
418
+ if not isinstance(positions, dict) or set(positions) != set(names):
419
+ raise ArtifactIntegrityError("predicate selection has different input tables")
420
+ candidate_items: list[tuple[str, int]] = []
421
+ for name in names:
422
+ indexes = positions[name]
423
+ if not isinstance(indexes, list):
424
+ raise ArtifactIntegrityError("invalid predicate row positions")
425
+ validated = tuple(
426
+ TypeAdapter(NonNegativeInt).validate_python(i, strict=True) for i in indexes
427
+ )
428
+ if tuple(sorted(set(validated))) != validated or any(
429
+ i >= original_rows[name] for i in validated
430
+ ):
431
+ raise ArtifactIntegrityError("invalid predicate row positions")
432
+ candidate_items.extend((name, i) for i in validated)
433
+ candidate = tuple(candidate_items)
434
+ baseline = evaluation <= result.options.confirmations
435
+ if not result.baseline_verified or baseline:
436
+ if candidate != initial:
437
+ raise ArtifactIntegrityError("baseline observations belong to different rows")
438
+ elif not set(candidate) < set(current) or (pending is not None and candidate != pending):
439
+ raise ArtifactIntegrityError(
440
+ "predicate candidate is not a deletion of the current case"
441
+ )
442
+ index += 1
443
+ if index >= len(events) - 1:
444
+ raise ArtifactIntegrityError("predicate attempt has no recorded outcome")
445
+ outcome = events[index]
446
+ if (outcome.stage, outcome.state, outcome.code) == (
447
+ "predicate",
448
+ "failed",
449
+ "predicate_failed",
450
+ ):
451
+ error_file = attempt + "error.json"
452
+ required_attempts.add(error_file)
453
+ if error_file not in listed or parse_json((directory / error_file).read_bytes()) != {
454
+ "code": "predicate_failed"
455
+ }:
456
+ raise ArtifactIntegrityError(
457
+ "predicate failure evidence is missing or inconsistent"
458
+ )
459
+ if result.status != "evaluation_failed" or index != len(events) - 2:
460
+ raise ArtifactIntegrityError("predicate failure contradicts the final status")
461
+ last_outcome = "predicate_failed"
462
+ index += 1
463
+ continue
464
+ observation_file = attempt + "observation.json"
465
+ required_attempts.add(observation_file)
466
+ if observation_file not in listed:
467
+ raise ArtifactIntegrityError("predicate observation is missing from the inventory")
468
+ observation = load_json(ReductionObservation, directory / observation_file)
469
+ matched = observation.preconditions == "satisfied" and (
470
+ observation.failure_signature == result.failure_signature
471
+ )
472
+ code = "matched" if matched else "not_preserved"
473
+ if (outcome.stage, outcome.state, outcome.code) != ("predicate", "finished", code):
474
+ raise ArtifactIntegrityError("predicate observation contradicts its journal outcome")
475
+ if result.baseline_verified and baseline:
476
+ if not matched:
477
+ raise ArtifactIntegrityError(
478
+ "verified baseline does not reproduce the target failure"
479
+ )
480
+ baseline_matches += 1
481
+ elif matched:
482
+ pending = candidate
483
+ matches += 1
484
+ if matches > result.options.confirmations:
485
+ raise ArtifactIntegrityError("predicate confirmations exceed the configured count")
486
+ else:
487
+ rejected.add(candidate)
488
+ pending = None
489
+ matches = 0
490
+ last_outcome = code
491
+ index += 1
492
+ if (
493
+ evaluation != result.evaluations
494
+ or accepted != result.accepted_candidates
495
+ or (result.baseline_verified and baseline_matches != result.options.confirmations)
496
+ or {item for item in listed if item.startswith("attempt-")} != required_attempts
497
+ or current
498
+ != tuple((table.name, i) for table in result.tables for i in table.retained_positions)
499
+ ):
500
+ raise ArtifactIntegrityError("reduction attempts contradict the final result")
501
+ if result.status == "invalid_baseline" and (
502
+ result.baseline_verified or last_outcome != "not_preserved"
503
+ ):
504
+ raise ArtifactIntegrityError("invalid baseline status contradicts its observations")
505
+ if result.status == "evaluation_failed" and last_outcome != "predicate_failed":
506
+ raise ArtifactIntegrityError("evaluation failure lacks a failed predicate attempt")
507
+ if result.status in ("reduced", "unchanged"):
508
+ if (
509
+ not result.baseline_verified
510
+ or pending is not None
511
+ or ((result.status == "reduced") != bool(accepted))
512
+ ):
513
+ raise ArtifactIntegrityError("successful reduction status contradicts its search")
514
+ if result.minimality == "one_minimal" and any(
515
+ current[:offset] + current[offset + 1 :] not in rejected
516
+ for offset in range(len(current))
517
+ ):
518
+ raise ArtifactIntegrityError("one-minimal claim lacks the individual deletion checks")
equiframe/registry.py ADDED
@@ -0,0 +1,121 @@
1
+ """Explicit instance-owned registry of trusted native implementations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Callable, Mapping
6
+ from dataclasses import dataclass
7
+ from typing import TYPE_CHECKING, Literal, Protocol
8
+
9
+ from pydantic import ConfigDict, TypeAdapter
10
+
11
+ from equiframe._base import Identifier, PositiveInt, Text
12
+ from equiframe.errors import InvalidExecutionInput
13
+ from equiframe.execution_types import ExecutionPlan
14
+
15
+ if TYPE_CHECKING:
16
+ import pandas as pd
17
+ import polars as pl
18
+
19
+ from equiframe.execution import RunContext
20
+
21
+
22
+ class Transformation(Protocol):
23
+ @property
24
+ def engine(self) -> str: ...
25
+
26
+ @property
27
+ def transformation_id(self) -> str: ...
28
+
29
+ @property
30
+ def revision(self) -> int: ...
31
+
32
+ @property
33
+ def code_version(self) -> str: ...
34
+
35
+
36
+ def _native_identity(
37
+ engine: str, expected: str, transformation_id: str, revision: int, code_version: str
38
+ ) -> None:
39
+ """Validate built-in metadata without importing or invoking native runtimes."""
40
+ safe_errors = ConfigDict(hide_input_in_errors=True)
41
+ TypeAdapter(Identifier, config=safe_errors).validate_python(transformation_id, strict=True)
42
+ TypeAdapter(PositiveInt, config=safe_errors).validate_python(revision, strict=True)
43
+ TypeAdapter(Text, config=safe_errors).validate_python(code_version, strict=True)
44
+ if type(engine) is not str or engine != expected:
45
+ raise InvalidExecutionInput("native transformation engine differs from its class")
46
+
47
+
48
+ @dataclass(frozen=True)
49
+ class PandasTransformation:
50
+ transformation_id: str
51
+ callback: Callable[[Mapping[str, pd.DataFrame], RunContext], pd.DataFrame]
52
+ code_version: str
53
+ revision: int = 1
54
+ engine: Literal["pandas"] = "pandas"
55
+
56
+ def __post_init__(self) -> None:
57
+ _native_identity(
58
+ self.engine, "pandas", self.transformation_id, self.revision, self.code_version
59
+ )
60
+ if not callable(self.callback):
61
+ raise InvalidExecutionInput("native transformation requires a trusted callable")
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class PolarsTransformation:
66
+ transformation_id: str
67
+ callback: Callable[[Mapping[str, pl.DataFrame], RunContext], pl.DataFrame | pl.LazyFrame]
68
+ code_version: str
69
+ revision: int = 1
70
+ engine: Literal["polars"] = "polars"
71
+
72
+ def __post_init__(self) -> None:
73
+ _native_identity(
74
+ self.engine, "polars", self.transformation_id, self.revision, self.code_version
75
+ )
76
+ if not callable(self.callback):
77
+ raise InvalidExecutionInput("native transformation requires a trusted callable")
78
+
79
+
80
+ @dataclass(frozen=True)
81
+ class DuckDBTransformation:
82
+ transformation_id: str
83
+ sql: str
84
+ code_version: str
85
+ revision: int = 1
86
+ engine: Literal["duckdb"] = "duckdb"
87
+
88
+ def __post_init__(self) -> None:
89
+ _native_identity(
90
+ self.engine, "duckdb", self.transformation_id, self.revision, self.code_version
91
+ )
92
+ if not isinstance(self.sql, str) or not self.sql.strip() or "\x00" in self.sql:
93
+ raise InvalidExecutionInput("native transformation requires nonempty trusted SQL text")
94
+ if any(0xD800 <= ord(char) <= 0xDFFF for char in self.sql):
95
+ raise InvalidExecutionInput("native SQL requires valid Unicode scalar values")
96
+
97
+
98
+ class TransformationRegistry:
99
+ def __init__(self) -> None:
100
+ self._items: dict[tuple[str, str, int], Transformation] = {}
101
+
102
+ def register(self, transformation: Transformation) -> None:
103
+ from pydantic import TypeAdapter
104
+
105
+ TypeAdapter(Identifier).validate_python(transformation.engine, strict=True)
106
+ TypeAdapter(Identifier).validate_python(transformation.transformation_id, strict=True)
107
+ TypeAdapter(PositiveInt).validate_python(transformation.revision, strict=True)
108
+ TypeAdapter(Text).validate_python(transformation.code_version, strict=True)
109
+ key = (transformation.engine, transformation.transformation_id, transformation.revision)
110
+ if key in self._items:
111
+ raise InvalidExecutionInput("transformation already registered")
112
+ self._items[key] = transformation
113
+
114
+ def resolve(self, plan: ExecutionPlan) -> Transformation:
115
+ key = (plan.engine, plan.transformation_id, plan.transformation_revision)
116
+ try:
117
+ return self._items[key]
118
+ except KeyError:
119
+ raise InvalidExecutionInput(
120
+ "transformation is not registered for this engine/revision"
121
+ ) from None