equiframe 0.12.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. equiframe/__init__.py +832 -0
  2. equiframe/__main__.py +3 -0
  3. equiframe/_base.py +84 -0
  4. equiframe/_cli_support.py +125 -0
  5. equiframe/_comparison_io.py +161 -0
  6. equiframe/_dataset_provenance.py +146 -0
  7. equiframe/_decimal.py +22 -0
  8. equiframe/_diff_capture.py +433 -0
  9. equiframe/_doctor.py +123 -0
  10. equiframe/_document_io.py +61 -0
  11. equiframe/_documents.py +79 -0
  12. equiframe/_duckdb_transport.py +149 -0
  13. equiframe/_error_reporting.py +52 -0
  14. equiframe/_first_use.py +102 -0
  15. equiframe/_path_compat.py +26 -0
  16. equiframe/_physical.py +133 -0
  17. equiframe/_r_review.R +308 -0
  18. equiframe/_regular_io.py +167 -0
  19. equiframe/_report_format_v1.py +298 -0
  20. equiframe/_report_io.py +591 -0
  21. equiframe/_stability_inventory.py +1421 -0
  22. equiframe/_timezones.py +257 -0
  23. equiframe/_vendor/cyclonedx/LICENSE +201 -0
  24. equiframe/_vendor/cyclonedx/PROVENANCE.md +18 -0
  25. equiframe/_vendor/cyclonedx/bom-1.6.schema.json +5673 -0
  26. equiframe/_vendor/cyclonedx/jsf-0.82.schema.json +240 -0
  27. equiframe/_vendor/cyclonedx/spdx.schema.json +712 -0
  28. equiframe/adapters.py +454 -0
  29. equiframe/assertion_types.py +122 -0
  30. equiframe/assertions.py +271 -0
  31. equiframe/benchmark.py +101 -0
  32. equiframe/bundle.py +240 -0
  33. equiframe/cli.py +693 -0
  34. equiframe/comparison.py +937 -0
  35. equiframe/comparison_report.py +660 -0
  36. equiframe/comparison_types.py +257 -0
  37. equiframe/compatibility.py +546 -0
  38. equiframe/contracts.py +879 -0
  39. equiframe/datasets.py +1301 -0
  40. equiframe/decision.py +34 -0
  41. equiframe/destinations.py +295 -0
  42. equiframe/detector_types.py +431 -0
  43. equiframe/detectors.py +389 -0
  44. equiframe/diff.py +192 -0
  45. equiframe/diff_contracts.py +523 -0
  46. equiframe/docs/adoption.md +329 -0
  47. equiframe/docs/api.md +534 -0
  48. equiframe/docs/arquitecto.md +143 -0
  49. equiframe/docs/choosing-api.md +84 -0
  50. equiframe/docs/cientifico.md +180 -0
  51. equiframe/docs/compatibility.md +254 -0
  52. equiframe/docs/contributing.md +171 -0
  53. equiframe/docs/dependencies.md +237 -0
  54. equiframe/docs/eda.md +166 -0
  55. equiframe/docs/extending.md +158 -0
  56. equiframe/docs/fast-ci.md +52 -0
  57. equiframe/docs/first-use.md +78 -0
  58. equiframe/docs/index.md +142 -0
  59. equiframe/docs/ingestion.md +145 -0
  60. equiframe/docs/inicio.md +129 -0
  61. equiframe/docs/installed-private.md +97 -0
  62. equiframe/docs/journeys.md +362 -0
  63. equiframe/docs/lectura-conversion.md +211 -0
  64. equiframe/docs/licensing.md +17 -0
  65. equiframe/docs/maintenance.md +233 -0
  66. equiframe/docs/migration-ir.md +333 -0
  67. equiframe/docs/option-audit.md +45 -0
  68. equiframe/docs/outcome-gates.md +134 -0
  69. equiframe/docs/performance.md +263 -0
  70. equiframe/docs/piloto-simulado.md +56 -0
  71. equiframe/docs/quickstart.md +156 -0
  72. equiframe/docs/r_review.md +212 -0
  73. equiframe/docs/recipes.md +248 -0
  74. equiframe/docs/releasing.md +183 -0
  75. equiframe/docs/security.md +218 -0
  76. equiframe/docs/semantics.md +274 -0
  77. equiframe/docs/sqlite-snapshots.md +262 -0
  78. equiframe/docs/universal-sources.md +49 -0
  79. equiframe/docs/universalidad-cinco-pasos.md +107 -0
  80. equiframe/docs/upgrading.md +538 -0
  81. equiframe/duckdb_capture.py +410 -0
  82. equiframe/eda/__init__.py +36 -0
  83. equiframe/eda/__main__.py +3 -0
  84. equiframe/eda/_backend.py +612 -0
  85. equiframe/eda/_inventory.py +49 -0
  86. equiframe/eda/_paths.py +49 -0
  87. equiframe/eda/_report.py +291 -0
  88. equiframe/eda/api.py +535 -0
  89. equiframe/eda/cli.py +102 -0
  90. equiframe/eda/models.py +239 -0
  91. equiframe/errors.py +83 -0
  92. equiframe/evidence.py +375 -0
  93. equiframe/examples/integrations/eda_generic.py +60 -0
  94. equiframe/examples/integrations/outcome_gates.py +94 -0
  95. equiframe/examples/integrations/sqlite_snapshot.py +246 -0
  96. equiframe/examples/integrations/sqlite_snapshot_demo.py +226 -0
  97. equiframe/examples/integrations/universal_first_use.py +296 -0
  98. equiframe/examples/journeys/README.md +31 -0
  99. equiframe/examples/journeys/__init__.py +1 -0
  100. equiframe/examples/journeys/a_pandas_refactor.py +88 -0
  101. equiframe/examples/journeys/b_native_migration.py +181 -0
  102. equiframe/examples/journeys/c_retail_etl.py +460 -0
  103. equiframe/examples/journeys/common.py +136 -0
  104. equiframe/examples/journeys/d_financial_reconciliation.py +96 -0
  105. equiframe/examples/journeys/e_telemetry.py +135 -0
  106. equiframe/examples/journeys/f_data_science_preparation.py +84 -0
  107. equiframe/examples/journeys/g_external_snapshots.py +197 -0
  108. equiframe/examples/journeys/h_portable_ci.py +129 -0
  109. equiframe/examples/journeys/requirements.lock +249 -0
  110. equiframe/examples/journeys/verify_journeys.py +117 -0
  111. equiframe/examples/load_convert_compare.py +85 -0
  112. equiframe/execution.py +355 -0
  113. equiframe/execution_types.py +177 -0
  114. equiframe/ingestion.py +744 -0
  115. equiframe/local.py +126 -0
  116. equiframe/migration/__init__.py +94 -0
  117. equiframe/migration/_physical.py +8 -0
  118. equiframe/migration/backends/_timestamp_transport.py +109 -0
  119. equiframe/migration/backends/duckdb.py +651 -0
  120. equiframe/migration/backends/duckdb_decimal.py +125 -0
  121. equiframe/migration/backends/duckdb_time.py +87 -0
  122. equiframe/migration/backends/pandas.py +544 -0
  123. equiframe/migration/backends/polars.py +915 -0
  124. equiframe/migration/backends/polars_decimal.py +231 -0
  125. equiframe/migration/io.py +198 -0
  126. equiframe/migration/ports.py +96 -0
  127. equiframe/migration/runner.py +545 -0
  128. equiframe/migration/runner_types.py +405 -0
  129. equiframe/migration/types.py +354 -0
  130. equiframe/migration/validation.py +619 -0
  131. equiframe/normalization.py +358 -0
  132. equiframe/pipeline.py +274 -0
  133. equiframe/pipeline_types.py +247 -0
  134. equiframe/ports.py +39 -0
  135. equiframe/preprocessing.py +110 -0
  136. equiframe/profile_types.py +343 -0
  137. equiframe/profiles.py +447 -0
  138. equiframe/py.typed +0 -0
  139. equiframe/r_review.py +706 -0
  140. equiframe/reduction.py +518 -0
  141. equiframe/registry.py +121 -0
  142. equiframe/report_rendering.py +366 -0
  143. equiframe/report_types.py +294 -0
  144. equiframe/reporting.py +26 -0
  145. equiframe/rule_types.py +178 -0
  146. equiframe/rules.py +819 -0
  147. equiframe/schemas/r-execution-binding-1.0.schema.json +46 -0
  148. equiframe/schemas/r-review-binding-1.0.schema.json +28 -0
  149. equiframe/schemas/r-review-options-1.0.schema.json +38 -0
  150. equiframe/schemas/r-review-result-1.0.schema.json +207 -0
  151. equiframe/schemas/reviewed-decision-1.0.schema.json +3394 -0
  152. equiframe/serialization.py +152 -0
  153. equiframe/snapshot.py +296 -0
  154. equiframe/sources.py +283 -0
  155. equiframe/tables.py +459 -0
  156. equiframe/types.py +71 -0
  157. equiframe/values.py +252 -0
  158. equiframe/verification_request.py +35 -0
  159. equiframe/verify.py +799 -0
  160. equiframe/verify_types.py +255 -0
  161. equiframe/workflow.py +510 -0
  162. equiframe/workflow_request.py +38 -0
  163. equiframe-0.12.4.dist-info/METADATA +268 -0
  164. equiframe-0.12.4.dist-info/RECORD +170 -0
  165. equiframe-0.12.4.dist-info/WHEEL +4 -0
  166. equiframe-0.12.4.dist-info/entry_points.txt +3 -0
  167. equiframe-0.12.4.dist-info/licenses/LICENSE +21 -0
  168. equiframe-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md +42 -0
  169. equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/LICENSE +201 -0
  170. equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/PROVENANCE.md +18 -0
equiframe/rules.py ADDED
@@ -0,0 +1,819 @@
1
+ """Operational rules over one captured execution; never invokes a comparator or ETL."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import inspect
7
+ import json
8
+ import os
9
+ import tempfile
10
+ import time
11
+ from collections.abc import Callable, Iterator, Mapping
12
+ from dataclasses import dataclass
13
+ from datetime import datetime
14
+ from datetime import timezone as _timezone
15
+ from decimal import Decimal, localcontext
16
+ from fractions import Fraction
17
+ from pathlib import Path
18
+ from typing import TYPE_CHECKING, BinaryIO, Literal
19
+ from uuid import uuid4
20
+
21
+ from equiframe._decimal import exact_decimal_context
22
+ from equiframe._document_io import read_document
23
+ from equiframe._error_reporting import report_failure
24
+ from equiframe._regular_io import local_input_path, local_output_directory
25
+ from equiframe.comparison_types import ComparisonPolicy, ComparisonValue
26
+ from equiframe.contracts import (
27
+ AggregateCheck,
28
+ CardinalityCheck,
29
+ ColumnKeyScope,
30
+ Contract,
31
+ JoinCheck,
32
+ KeyCheck,
33
+ OrderCheck,
34
+ RuleDefinition,
35
+ SchemaCheck,
36
+ SignificantOrder,
37
+ TableSpec,
38
+ )
39
+ from equiframe.errors import (
40
+ ArtifactIntegrityError,
41
+ CapabilityError,
42
+ ExecutionCancelled,
43
+ ExecutionTimeout,
44
+ ResourceLimitError,
45
+ )
46
+ from equiframe.execution import CancellationToken
47
+ from equiframe.execution_types import ExecutionResult
48
+ from equiframe.local import LocalArtifactStore, local_path
49
+ from equiframe.normalization import (
50
+ equal,
51
+ identity,
52
+ normalize,
53
+ number,
54
+ order_value,
55
+ physical_value,
56
+ schema_compatible,
57
+ )
58
+ from equiframe.rule_types import (
59
+ CheckResult,
60
+ CheckState,
61
+ RuleCoverage,
62
+ RuleEvaluation,
63
+ RuleFinding,
64
+ RuleOptions,
65
+ )
66
+ from equiframe.serialization import _constant, _JsonObject, _unique_objects, contract_fingerprint
67
+ from equiframe.tables import read_table
68
+ from equiframe.types import ArtifactRef, DecisionState
69
+
70
+ UTC = _timezone.utc
71
+
72
+ if TYPE_CHECKING:
73
+ import pyarrow as pa
74
+
75
+ RuleFunction = Callable[[RuleDefinition, "RuleContext"], None]
76
+ RuleKind = Literal["schema", "key", "cardinality", "join", "order", "aggregate"]
77
+
78
+
79
+ def _json(value: object) -> bytes:
80
+ return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode(
81
+ "ascii"
82
+ )
83
+
84
+
85
+ def _ref(path: Path) -> ArtifactRef:
86
+ data = path.read_bytes()
87
+ return ArtifactRef(
88
+ schema_version="1.0",
89
+ uri=path.resolve().as_uri(),
90
+ sha256=hashlib.sha256(data).hexdigest(),
91
+ format="jsonl",
92
+ media_type="application/x-ndjson",
93
+ size_bytes=len(data),
94
+ )
95
+
96
+
97
+ @dataclass(frozen=True)
98
+ class RuleImplementation:
99
+ kind: RuleKind
100
+ revision: int
101
+ identity: str
102
+ evaluate: RuleFunction
103
+
104
+
105
+ class RuleRegistry:
106
+ """Explicit trusted implementations; source bytes and revision enter the freeze hash."""
107
+
108
+ def __init__(self) -> None:
109
+ self._items: dict[RuleKind, RuleImplementation] = {}
110
+
111
+ def register(self, kind: RuleKind, revision: int, evaluate: RuleFunction) -> None:
112
+ if kind in self._items or revision < 1:
113
+ raise ValueError("duplicate rule kind or invalid implementation revision")
114
+ try:
115
+ source_path = inspect.getsourcefile(evaluate)
116
+ if source_path is None:
117
+ raise OSError("source path unavailable")
118
+ source = Path(source_path).read_bytes()
119
+ # Built-in semantic helpers are part of R's implementation identity.
120
+ if evaluate.__module__ == __name__:
121
+ helper_path = inspect.getsourcefile(normalize)
122
+ if helper_path is None:
123
+ raise OSError("normalization source path unavailable")
124
+ source += Path(helper_path).read_bytes()
125
+ except (OSError, TypeError) as error:
126
+ raise ValueError("rule implementation must have inspectable source") from error
127
+ digest = hashlib.sha256(source).hexdigest()
128
+ name = f"{evaluate.__module__}.{evaluate.__qualname__}"
129
+ self._items[kind] = RuleImplementation(
130
+ kind, revision, f"{name}@{revision}:{digest}", evaluate
131
+ )
132
+
133
+ def resolve(self, kind: RuleKind) -> RuleImplementation | None:
134
+ return self._items.get(kind)
135
+
136
+ def identities(self) -> tuple[tuple[str, str], ...]:
137
+ return tuple(sorted((kind, item.identity) for kind, item in self._items.items()))
138
+
139
+
140
+ def default_rule_registry() -> RuleRegistry:
141
+ result = RuleRegistry()
142
+ implementations: tuple[tuple[RuleKind, RuleFunction], ...] = (
143
+ ("schema", _schema),
144
+ ("key", _key),
145
+ ("cardinality", _cardinality),
146
+ ("join", _join),
147
+ ("order", _order),
148
+ ("aggregate", _aggregate),
149
+ )
150
+ for kind, function in implementations:
151
+ result.register(kind, 1, function)
152
+ return result
153
+
154
+
155
+ def rule_set_fingerprint(contract: Contract, registry: RuleRegistry, options: RuleOptions) -> str:
156
+ """Freeze exact contract, rule code, and resource/configuration choices."""
157
+ return hashlib.sha256(
158
+ _json(
159
+ {
160
+ "contract_sha256": contract_fingerprint(contract),
161
+ "implementations": registry.identities(),
162
+ "options": options.model_dump(mode="json"),
163
+ "format": "equiframe-rules-1.0",
164
+ }
165
+ )
166
+ ).hexdigest()
167
+
168
+
169
+ class RuleContext:
170
+ """Only declared contract and verified tables are visible to a rule implementation."""
171
+
172
+ def __init__(
173
+ self,
174
+ contract: Contract,
175
+ tables: Mapping[str, pa.Table],
176
+ options: RuleOptions,
177
+ cancellation: CancellationToken,
178
+ started: float,
179
+ ) -> None:
180
+ self.contract, self.tables, self.options = contract, tables, options
181
+ self.cancellation, self.started = cancellation, started
182
+ self.work = self.examined = self.violations = self.evidence_bytes = 0
183
+ self.examples: list[RuleFinding] = []
184
+ self.check_id = ""
185
+ self.sink: BinaryIO | None = None
186
+
187
+ def checkpoint(self, cost: int = 1) -> None:
188
+ self.work += cost
189
+ if self.cancellation.cancelled:
190
+ raise ExecutionCancelled("rule evaluation cancelled at checkpoint")
191
+ if (
192
+ self.options.timeout_seconds is not None
193
+ and time.perf_counter() - self.started >= self.options.timeout_seconds
194
+ ):
195
+ raise ExecutionTimeout("cooperative rule evaluation timeout")
196
+ if self.work > self.options.max_work:
197
+ raise ResourceLimitError("rule max_work exhausted")
198
+
199
+ def table(self, name: str) -> pa.Table:
200
+ self.checkpoint()
201
+ if name not in self.tables:
202
+ raise CapabilityError("captured table unavailable: " + name)
203
+ return self.tables[name]
204
+
205
+ def spec(self, name: str) -> TableSpec:
206
+ return next(t for t in (*self.contract.inputs, self.contract.output) if t.name == name)
207
+
208
+ def rows(
209
+ self, name: str, columns: tuple[str, ...]
210
+ ) -> Iterator[tuple[int, tuple[ComparisonValue, ...]]]:
211
+ table = self.table(name)
212
+ missing = set(columns) - set(table.column_names)
213
+ if missing:
214
+ raise CapabilityError("missing columns in " + name + ": " + ", ".join(sorted(missing)))
215
+ declared = {column.name: column for column in self.spec(name).schema_spec.columns}
216
+ for column_name in columns:
217
+ if not schema_compatible(
218
+ table.schema.field(column_name).type, declared[column_name], ComparisonPolicy()
219
+ ):
220
+ raise CapabilityError("incompatible captured type: " + name + "." + column_name)
221
+ for position in range(table.num_rows):
222
+ self.checkpoint()
223
+ self.examined += 1
224
+ if self.examined > self.options.max_rows:
225
+ raise ResourceLimitError("rule max_rows exhausted")
226
+ values = tuple(
227
+ normalize(physical_value(table[name][position]), declared[name])[0]
228
+ for name in columns
229
+ )
230
+ yield position, values
231
+
232
+ def emit(
233
+ self,
234
+ code: str,
235
+ table: str,
236
+ *,
237
+ position: int | None = None,
238
+ column: str | None = None,
239
+ observed: tuple[ComparisonValue, ...] = (),
240
+ expected: tuple[ComparisonValue, ...] = (),
241
+ message: str | None = None,
242
+ ) -> None:
243
+ finding = RuleFinding(
244
+ check_id=self.check_id,
245
+ code=code,
246
+ table=table,
247
+ position=position,
248
+ column=column,
249
+ observed=observed,
250
+ expected=expected,
251
+ message=message or code.replace("_", " "),
252
+ )
253
+ if not (self.sink is not None):
254
+ raise AssertionError("Internal invariant check failed.")
255
+ block = _json(finding.model_dump(mode="json")) + b"\n"
256
+ if self.evidence_bytes + len(block) > self.options.max_evidence_bytes:
257
+ raise ResourceLimitError("rule max_evidence_bytes exhausted")
258
+ self.sink.write(block)
259
+ self.evidence_bytes += len(block)
260
+ self.violations += 1
261
+ if len(self.examples) < self.options.max_examples:
262
+ self.examples.append(finding)
263
+ self.checkpoint()
264
+
265
+
266
+ def _schema(definition: RuleDefinition, context: RuleContext) -> None:
267
+ if not (isinstance(definition, SchemaCheck)):
268
+ raise AssertionError("Internal invariant check failed.")
269
+ table = context.table(definition.table)
270
+ columns = context.spec(definition.table).schema_spec.columns
271
+ expected_names = tuple(column.name for column in columns)
272
+ if tuple(table.column_names) != expected_names:
273
+ context.emit(
274
+ "schema_columns",
275
+ definition.table,
276
+ message=f"expected columns {expected_names!r}; observed {tuple(table.column_names)!r}",
277
+ )
278
+ for column in columns:
279
+ if column.name not in table.column_names:
280
+ continue
281
+ try:
282
+ compatible = schema_compatible(
283
+ table.schema.field(column.name).type, column, ComparisonPolicy()
284
+ )
285
+ except CapabilityError:
286
+ compatible = False
287
+ if not compatible:
288
+ context.emit(
289
+ "schema_type",
290
+ definition.table,
291
+ column=column.name,
292
+ message=(
293
+ f"expected {column.logical_type!r}; "
294
+ f"observed {table.schema.field(column.name).type}"
295
+ ),
296
+ )
297
+ continue
298
+ for position, values in context.rows(definition.table, (column.name,)):
299
+ _, issues = normalize(physical_value(table[column.name][position]), column)
300
+ for issue in issues:
301
+ context.emit(
302
+ issue, definition.table, position=position, column=column.name, observed=values
303
+ )
304
+
305
+
306
+ def _key(definition: RuleDefinition, context: RuleContext) -> None:
307
+ if not (isinstance(definition, KeyCheck)):
308
+ raise AssertionError("Internal invariant check failed.")
309
+ key = next(
310
+ k for k in context.spec(definition.table).schema_spec.keys if k.name == definition.key
311
+ )
312
+ scope = key.scope.columns if isinstance(key.scope, ColumnKeyScope) else ()
313
+ seen: dict[tuple[object, ...], int] = {}
314
+ for position, values in context.rows(definition.table, (*scope, *key.columns)):
315
+ scoped, keyed = values[: len(scope)], values[len(scope) :]
316
+ if any(value.kind == "null" for value in keyed):
317
+ if key.incomplete == "reject":
318
+ context.emit("incomplete_key", definition.table, position=position, observed=keyed)
319
+ continue
320
+ token = tuple(identity(v) for v in (*scoped, *keyed))
321
+ if key.unique and token in seen:
322
+ context.emit(
323
+ "duplicate_key",
324
+ definition.table,
325
+ position=position,
326
+ observed=keyed,
327
+ message=f"duplicate key; first position {seen[token]}",
328
+ )
329
+ else:
330
+ seen[token] = position
331
+
332
+
333
+ def _cardinality(definition: RuleDefinition, context: RuleContext) -> None:
334
+ if not (isinstance(definition, CardinalityCheck)):
335
+ raise AssertionError("Internal invariant check failed.")
336
+ count = context.table(definition.table).num_rows
337
+ if count < definition.minimum or (
338
+ definition.maximum is not None and count > definition.maximum
339
+ ):
340
+ context.emit(
341
+ "row_count",
342
+ definition.table,
343
+ observed=(ComparisonValue(kind="integer", value=str(count)),),
344
+ message=f"row count {count}; expected {definition.minimum}..{definition.maximum}",
345
+ )
346
+
347
+
348
+ def _join(definition: RuleDefinition, context: RuleContext) -> None:
349
+ if not (isinstance(definition, JoinCheck)):
350
+ raise AssertionError("Internal invariant check failed.")
351
+ sides: list[dict[tuple[object, ...], list[int]]] = []
352
+ for name, columns in (
353
+ (definition.left_table, definition.left_columns),
354
+ (definition.right_table, definition.right_columns),
355
+ ):
356
+ groups: dict[tuple[object, ...], list[int]] = {}
357
+ for position, values in context.rows(name, columns):
358
+ if any(value.kind == "null" for value in values):
359
+ if definition.unmatched == "reject":
360
+ context.emit("unmatched_null_key", name, position=position, observed=values)
361
+ continue
362
+ groups.setdefault(tuple(identity(v) for v in values), []).append(position)
363
+ sides.append(groups)
364
+ left, right = sides
365
+ unique_left = definition.relationship in ("one_to_one", "one_to_many")
366
+ unique_right = definition.relationship in ("one_to_one", "many_to_one")
367
+ for name, groups, require_unique, other in (
368
+ (definition.left_table, left, unique_left, right),
369
+ (definition.right_table, right, unique_right, left),
370
+ ):
371
+ for token, positions in groups.items():
372
+ context.checkpoint()
373
+ if require_unique and len(positions) > 1:
374
+ context.emit(
375
+ "join_cardinality",
376
+ name,
377
+ position=positions[1],
378
+ message=f"key multiplicity {len(positions)}",
379
+ )
380
+ if definition.unmatched == "reject" and token not in other:
381
+ context.emit(
382
+ "unmatched_key",
383
+ name,
384
+ position=positions[0],
385
+ message=f"{len(positions)} rows have no partner",
386
+ )
387
+
388
+
389
+ def _order(definition: RuleDefinition, context: RuleContext) -> None:
390
+ if not (isinstance(definition, OrderCheck)):
391
+ raise AssertionError("Internal invariant check failed.")
392
+ policy = context.spec(definition.table).schema_spec.order
393
+ if not (isinstance(policy, SignificantOrder)):
394
+ raise AssertionError("Internal invariant check failed.")
395
+ previous: tuple[ComparisonValue, ...] | None = None
396
+ for position, values in context.rows(definition.table, definition.columns):
397
+ if previous is not None:
398
+ direction = 0
399
+ for old, new, sort in zip(previous, values, policy.by, strict=True):
400
+ if old.kind == "null" or new.kind == "null":
401
+ if old.kind == new.kind:
402
+ continue
403
+ direction = -1 if (old.kind == "null") == (sort.nulls == "first") else 1
404
+ else:
405
+ a, b = order_value(old), order_value(new)
406
+ direction = -1 if a < b else 1 if a > b else 0
407
+ if sort.direction == "descending":
408
+ direction = -direction
409
+ if direction:
410
+ break
411
+ if direction > 0:
412
+ context.emit(
413
+ "out_of_order",
414
+ definition.table,
415
+ position=position,
416
+ observed=values,
417
+ expected=previous,
418
+ )
419
+ elif direction == 0 and policy.ties == "reject":
420
+ context.emit("order_tie", definition.table, position=position, observed=values)
421
+ previous = values
422
+
423
+
424
+ def _aggregate(definition: RuleDefinition, context: RuleContext) -> None:
425
+ if not (isinstance(definition, AggregateCheck)):
426
+ raise AssertionError("Internal invariant check failed.")
427
+ source_columns = (*definition.group_by, *((definition.measure,) if definition.measure else ()))
428
+ groups: dict[tuple[object, ...], tuple[tuple[ComparisonValue, ...], list[ComparisonValue]]] = {}
429
+ source_rows = 0
430
+ for _, values in context.rows(definition.table, source_columns):
431
+ source_rows += 1
432
+ group_values = values[: len(definition.group_by)]
433
+ token = tuple(identity(v) for v in group_values)
434
+ if token not in groups:
435
+ groups[token] = group_values, []
436
+ if definition.measure is None or values[-1].kind != "null":
437
+ groups[token][1].append(
438
+ values[-1] if definition.measure else ComparisonValue(kind="integer", value="1")
439
+ )
440
+ if not definition.group_by and source_rows == 0:
441
+ groups[()] = (), []
442
+ observed: dict[tuple[object, ...], tuple[int, ComparisonValue]] = {}
443
+ for position, values in context.rows(
444
+ definition.output_table, (*definition.group_by, definition.output_column)
445
+ ):
446
+ token = tuple(identity(v) for v in values[:-1])
447
+ if token in observed:
448
+ context.emit(
449
+ "duplicate_group", definition.output_table, position=position, observed=values
450
+ )
451
+ observed[token] = position, values[-1]
452
+ if token not in groups:
453
+ context.emit(
454
+ "unexpected_group", definition.output_table, position=position, observed=values
455
+ )
456
+ target = next(
457
+ c
458
+ for c in context.spec(definition.output_table).schema_spec.columns
459
+ if c.name == definition.output_column
460
+ )
461
+ for group_token, (group_values, members) in groups.items():
462
+ context.checkpoint()
463
+ if group_token not in observed:
464
+ context.emit("missing_group", definition.output_table, observed=group_values)
465
+ continue
466
+ position, actual = observed[group_token]
467
+ expected: ComparisonValue
468
+ if definition.function == "count":
469
+ expected = ComparisonValue(kind="integer", value=str(len(members)))
470
+ elif not members:
471
+ expected = ComparisonValue(kind="null")
472
+ elif definition.function in ("min", "max"):
473
+ expected = (min if definition.function == "min" else max)(members, key=order_value)
474
+ else:
475
+ if members[0].kind == "float":
476
+ if target.numeric_policy is None or target.numeric_policy.tolerance is None:
477
+ raise CapabilityError(
478
+ "float sum requires a declared tolerance to account for rounding"
479
+ )
480
+ exact = sum((number(v) for v in members), Fraction())
481
+ if actual.kind != "float":
482
+ context.emit(
483
+ "aggregate_value",
484
+ definition.output_table,
485
+ position=position,
486
+ observed=(actual,),
487
+ )
488
+ continue
489
+ got = number(actual)
490
+ tolerance = target.numeric_policy.tolerance
491
+ bound = max(
492
+ Fraction(tolerance.absolute or 0),
493
+ Fraction(tolerance.relative or 0) * max(abs(exact), abs(got)),
494
+ )
495
+ if abs(exact - got) > bound:
496
+ context.emit(
497
+ "aggregate_value",
498
+ definition.output_table,
499
+ position=position,
500
+ observed=(actual,),
501
+ message="float sum outside declared tolerance",
502
+ )
503
+ continue
504
+ if members[0].kind == "integer":
505
+ expected = ComparisonValue(
506
+ kind="integer", value=str(sum(int(v.value or "0") for v in members))
507
+ )
508
+ else:
509
+ with localcontext(exact_decimal_context()) as decimal_context:
510
+ decimal_context.prec = 1000
511
+ value = sum((Decimal(v.value or "0") for v in members), Decimal())
512
+ expected = ComparisonValue(kind="decimal", value=str(value))
513
+ if not equal(actual, expected, target):
514
+ context.emit(
515
+ "aggregate_value",
516
+ definition.output_table,
517
+ position=position,
518
+ observed=(actual,),
519
+ expected=(expected,),
520
+ )
521
+
522
+
523
+ def _verified_tables(execution: ExecutionResult, options: RuleOptions) -> dict[str, pa.Table]:
524
+ if execution.state != "completed":
525
+ raise CapabilityError("execution state is " + execution.state)
526
+ if execution.output is None:
527
+ raise CapabilityError("completed execution lacks output")
528
+ declared = {item.name for item in execution.plan.contract.inputs}
529
+ artifacts = (*execution.inputs, execution.output)
530
+ if (
531
+ len(execution.inputs) != len(declared)
532
+ or {item.name for item in execution.inputs} != declared
533
+ ):
534
+ raise CapabilityError("captured inputs do not match contract")
535
+ if execution.output.name != execution.plan.contract.output.name:
536
+ raise CapabilityError("captured output name does not match contract")
537
+ result: dict[str, pa.Table] = {}
538
+ total_bytes = 0
539
+ actual_bytes = 0
540
+ decoded_bytes = 0
541
+ for artifact in artifacts:
542
+ total_bytes += artifact.data.size_bytes
543
+ actual_bytes += local_path(artifact.data).stat().st_size
544
+ if total_bytes > options.max_artifact_bytes or actual_bytes > options.max_artifact_bytes:
545
+ raise ResourceLimitError("rule max_artifact_bytes exhausted")
546
+ table = read_table(artifact.data)
547
+ decoded_bytes += table.nbytes
548
+ if decoded_bytes > options.max_artifact_bytes:
549
+ raise ResourceLimitError("decoded table exceeds rule max_artifact_bytes")
550
+ if table.num_rows != artifact.row_count:
551
+ raise ArtifactIntegrityError("captured row count does not match table")
552
+ if tuple(table.column_names) != tuple(column.name for column in artifact.columns):
553
+ raise ArtifactIntegrityError("captured columns do not match table")
554
+ for column in artifact.columns:
555
+ array = table[column.name]
556
+ import pyarrow as pa
557
+ import pyarrow.compute as pc
558
+
559
+ nan_count = (
560
+ int(pc.sum(pc.is_nan(array)).as_py() or 0)
561
+ if pa.types.is_floating(array.type)
562
+ else 0
563
+ )
564
+ if (
565
+ str(array.type) != column.arrow_type
566
+ or array.null_count != column.null_count
567
+ or nan_count != column.nan_count
568
+ ):
569
+ raise ArtifactIntegrityError("captured physical column metadata mismatch")
570
+ if artifact.index is not None:
571
+ total_bytes += artifact.index.size_bytes
572
+ actual_bytes += local_path(artifact.index).stat().st_size
573
+ if (
574
+ total_bytes > options.max_artifact_bytes
575
+ or actual_bytes > options.max_artifact_bytes
576
+ ):
577
+ raise ResourceLimitError("rule max_artifact_bytes exhausted")
578
+ index = read_table(artifact.index)
579
+ decoded_bytes += index.nbytes
580
+ if decoded_bytes > options.max_artifact_bytes:
581
+ raise ResourceLimitError("decoded index exceeds rule max_artifact_bytes")
582
+ if index.num_rows != table.num_rows or tuple(index.column_names) != tuple(
583
+ c.name for c in artifact.index_columns
584
+ ):
585
+ raise ArtifactIntegrityError("captured index metadata mismatch")
586
+ result[artifact.name] = table
587
+ return result
588
+
589
+
590
+ def _check(
591
+ definition: RuleDefinition,
592
+ check_id: str,
593
+ *,
594
+ kind: Literal["precondition", "rule"],
595
+ context: RuleContext | None,
596
+ registry: RuleRegistry,
597
+ directory: Path,
598
+ revision: int | None = None,
599
+ required: bool | None = None,
600
+ severity: Literal["info", "warning", "error"] | None = None,
601
+ reason: str | None = None,
602
+ applicable: bool = True,
603
+ ) -> CheckResult:
604
+ implementation = registry.resolve(definition.kind)
605
+ state: CheckState
606
+ if not applicable:
607
+ state, complete = "not_applicable", True
608
+ elif reason or context is None or implementation is None:
609
+ state, complete = "not_evaluable", False
610
+ reason = reason or (
611
+ "missing implementation: " + definition.kind
612
+ if implementation is None
613
+ else "evidence unavailable"
614
+ )
615
+ else:
616
+ state, complete = "satisfied", True
617
+ before_work = context.work if context else 0
618
+ path = directory / (uuid4().hex + ".jsonl")
619
+ with path.open("xb") as sink:
620
+ if context and state == "satisfied" and implementation is not None:
621
+ context.check_id, context.sink = check_id, sink
622
+ context.examples, context.violations, context.examined = [], 0, 0
623
+ try:
624
+ context.checkpoint()
625
+ implementation.evaluate(definition, context)
626
+ except Exception as error:
627
+ reason, complete = report_failure(error, "rule_evaluation_failed"), False
628
+ state = "violated" if context.violations else "not_evaluable"
629
+ else:
630
+ state = "violated" if context.violations else "satisfied"
631
+ finally:
632
+ context.sink = None
633
+ sink.flush()
634
+ os.fsync(sink.fileno())
635
+ findings = _ref(path)
636
+ return CheckResult(
637
+ check_id=check_id,
638
+ revision=revision,
639
+ kind=kind,
640
+ implementation=implementation.identity if implementation else None,
641
+ state=state,
642
+ complete=complete,
643
+ required=required,
644
+ severity=severity,
645
+ reason=reason,
646
+ violation_count=context.violations
647
+ if context and applicable and before_work != context.work
648
+ else 0,
649
+ examined_rows=context.examined
650
+ if context and applicable and before_work != context.work
651
+ else 0,
652
+ examples=tuple(context.examples)
653
+ if context and applicable and before_work != context.work
654
+ else (),
655
+ findings=findings,
656
+ )
657
+
658
+
659
+ def evaluate_rules(
660
+ execution: ExecutionResult,
661
+ *,
662
+ registry: RuleRegistry | None = None,
663
+ options: RuleOptions | None = None,
664
+ work_dir: Path | None = None,
665
+ cancellation: CancellationToken | None = None,
666
+ ) -> RuleEvaluation:
667
+ """Evaluate only a single completed run's captured inputs/output; persist fresh evidence."""
668
+ execution = ExecutionResult.model_validate(execution)
669
+ contract = execution.plan.contract
670
+ if execution.contract_sha256 != contract_fingerprint(contract):
671
+ raise ValueError("execution contract fingerprint mismatch")
672
+ options = RuleOptions.model_validate(RuleOptions() if options is None else options)
673
+ registry = registry or default_rule_registry()
674
+ cancellation = cancellation or CancellationToken()
675
+ identifier = "rules_" + uuid4().hex
676
+ directory = (
677
+ local_output_directory(Path.cwd() / "equiframe-runs" if work_dir is None else work_dir)
678
+ / identifier
679
+ )
680
+ directory.mkdir(parents=True, exist_ok=False)
681
+ started_at = datetime.now(UTC).isoformat()
682
+ start = time.perf_counter()
683
+ context: RuleContext | None = None
684
+ diagnostic: str | None = None
685
+ try:
686
+ if cancellation.cancelled:
687
+ raise ExecutionCancelled("rule evaluation cancelled before artifact loading")
688
+ tables = _verified_tables(execution, options)
689
+ context = RuleContext(contract, tables, options, cancellation, start)
690
+ context.checkpoint()
691
+ except Exception as error:
692
+ diagnostic = report_failure(error, "rule_inputs_unavailable")
693
+ preconditions = tuple(
694
+ _check(
695
+ item.requirement,
696
+ item.precondition_id,
697
+ kind="precondition",
698
+ context=context,
699
+ registry=registry,
700
+ directory=directory,
701
+ reason=diagnostic,
702
+ )
703
+ for item in contract.preconditions
704
+ )
705
+ precondition_map = {item.check_id: item for item in preconditions}
706
+ rules: list[CheckResult] = []
707
+ for rule in contract.rules:
708
+ dependencies = [precondition_map[name] for name in rule.applicability]
709
+ unavailable = [
710
+ item.check_id
711
+ for item in dependencies
712
+ if item.state == "not_evaluable" or not item.complete
713
+ ]
714
+ false = [
715
+ item.check_id for item in dependencies if item.state in ("violated", "not_applicable")
716
+ ]
717
+ reason = diagnostic
718
+ if false:
719
+ reason = "verified false precondition: " + ", ".join(false)
720
+ elif unavailable:
721
+ reason = "unverified applicability: " + ", ".join(unavailable)
722
+ rules.append(
723
+ _check(
724
+ rule.specification,
725
+ rule.rule_id,
726
+ kind="rule",
727
+ context=context,
728
+ registry=registry,
729
+ directory=directory,
730
+ revision=rule.revision,
731
+ required=rule.required,
732
+ severity=rule.severity,
733
+ reason=reason,
734
+ applicable=not false,
735
+ )
736
+ )
737
+ has_alert = any(item.state == "violated" for item in rules)
738
+ complete = (
739
+ bool(rules)
740
+ and diagnostic is None
741
+ and all(item.complete for item in (*preconditions, *rules))
742
+ and any(item.state in ("satisfied", "violated") for item in rules)
743
+ )
744
+ decision: DecisionState = "alert" if has_alert else "no_alert" if complete else "not_evaluable"
745
+ checked = tuple(item for item in rules if item.state in ("satisfied", "violated"))
746
+ alerts = tuple(item for item in rules if item.state == "violated")
747
+ severity_rank = {"info": 1, "warning": 2, "error": 3}
748
+ highest = max(
749
+ (item.severity for item in alerts if item.severity is not None),
750
+ key=severity_rank.__getitem__,
751
+ default=None,
752
+ )
753
+ coverage = RuleCoverage(
754
+ declared=len(rules),
755
+ checked=len(checked),
756
+ skipped=sum(item.state == "not_applicable" for item in rules),
757
+ unavailable=sum(item.state == "not_evaluable" for item in rules),
758
+ required_declared=sum(item.required is True for item in rules),
759
+ required_checked=sum(item.required is True for item in checked),
760
+ alerts=len(alerts),
761
+ finding_count=sum(item.violation_count for item in rules),
762
+ preconditions_declared=len(preconditions),
763
+ preconditions_verified=sum(item.complete for item in preconditions),
764
+ preconditions_unavailable=sum(not item.complete for item in preconditions),
765
+ highest_alert_severity=highest,
766
+ )
767
+ result = RuleEvaluation(
768
+ evaluation_id=identifier,
769
+ run_id=execution.run_id,
770
+ execution_id=execution.execution_id,
771
+ contract_id=contract.contract_id,
772
+ contract_revision=contract.revision,
773
+ contract_sha256=execution.contract_sha256,
774
+ rule_set_sha256=rule_set_fingerprint(contract, registry, options),
775
+ execution_state=execution.state,
776
+ options=options,
777
+ decision=decision,
778
+ complete=complete,
779
+ preconditions=preconditions,
780
+ rules=tuple(rules),
781
+ coverage=coverage,
782
+ started_at=started_at,
783
+ elapsed_seconds=time.perf_counter() - start,
784
+ work_units=context.work if context else 0,
785
+ diagnostic=diagnostic or ("empty rule set" if not rules else None),
786
+ )
787
+ temporary = directory / "rules.json.part"
788
+ with temporary.open("xb") as output:
789
+ output.write(_json(result.model_dump(mode="json")))
790
+ output.flush()
791
+ os.fsync(output.fileno())
792
+ os.link(temporary, directory / "rules.json")
793
+ temporary.unlink()
794
+ return result
795
+
796
+
797
+ def read_rule_evaluation(path: Path) -> RuleEvaluation:
798
+ """Validate manifest structure; call iter_rule_findings to verify referenced bytes."""
799
+ raw = read_document(local_input_path(path))
800
+ return RuleEvaluation.model_validate_json(raw)
801
+
802
+
803
+ def iter_rule_findings(ref: ArtifactRef) -> Iterator[RuleFinding]:
804
+ """Verify all finding bytes before yielding typed locations."""
805
+ if ref.format != "jsonl":
806
+ raise ValueError("rule findings require JSONL")
807
+ with tempfile.TemporaryDirectory(prefix="equiframe-rule-findings-") as temporary:
808
+ path = Path(temporary) / "findings.jsonl"
809
+ LocalArtifactStore(Path(temporary)).get(ref, path)
810
+ with path.open("rb") as source:
811
+ for line in source:
812
+ _unique_objects(
813
+ json.loads(
814
+ line.decode("utf-8"),
815
+ object_pairs_hook=_JsonObject,
816
+ parse_constant=_constant,
817
+ )
818
+ )
819
+ yield RuleFinding.model_validate_json(line)