equiframe 0.12.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. equiframe/__init__.py +832 -0
  2. equiframe/__main__.py +3 -0
  3. equiframe/_base.py +84 -0
  4. equiframe/_cli_support.py +125 -0
  5. equiframe/_comparison_io.py +161 -0
  6. equiframe/_dataset_provenance.py +146 -0
  7. equiframe/_decimal.py +22 -0
  8. equiframe/_diff_capture.py +433 -0
  9. equiframe/_doctor.py +123 -0
  10. equiframe/_document_io.py +61 -0
  11. equiframe/_documents.py +79 -0
  12. equiframe/_duckdb_transport.py +149 -0
  13. equiframe/_error_reporting.py +52 -0
  14. equiframe/_first_use.py +102 -0
  15. equiframe/_path_compat.py +26 -0
  16. equiframe/_physical.py +133 -0
  17. equiframe/_r_review.R +308 -0
  18. equiframe/_regular_io.py +167 -0
  19. equiframe/_report_format_v1.py +298 -0
  20. equiframe/_report_io.py +591 -0
  21. equiframe/_stability_inventory.py +1421 -0
  22. equiframe/_timezones.py +257 -0
  23. equiframe/_vendor/cyclonedx/LICENSE +201 -0
  24. equiframe/_vendor/cyclonedx/PROVENANCE.md +18 -0
  25. equiframe/_vendor/cyclonedx/bom-1.6.schema.json +5673 -0
  26. equiframe/_vendor/cyclonedx/jsf-0.82.schema.json +240 -0
  27. equiframe/_vendor/cyclonedx/spdx.schema.json +712 -0
  28. equiframe/adapters.py +454 -0
  29. equiframe/assertion_types.py +122 -0
  30. equiframe/assertions.py +271 -0
  31. equiframe/benchmark.py +101 -0
  32. equiframe/bundle.py +240 -0
  33. equiframe/cli.py +693 -0
  34. equiframe/comparison.py +937 -0
  35. equiframe/comparison_report.py +660 -0
  36. equiframe/comparison_types.py +257 -0
  37. equiframe/compatibility.py +546 -0
  38. equiframe/contracts.py +879 -0
  39. equiframe/datasets.py +1301 -0
  40. equiframe/decision.py +34 -0
  41. equiframe/destinations.py +295 -0
  42. equiframe/detector_types.py +431 -0
  43. equiframe/detectors.py +389 -0
  44. equiframe/diff.py +192 -0
  45. equiframe/diff_contracts.py +523 -0
  46. equiframe/docs/adoption.md +329 -0
  47. equiframe/docs/api.md +534 -0
  48. equiframe/docs/arquitecto.md +143 -0
  49. equiframe/docs/choosing-api.md +84 -0
  50. equiframe/docs/cientifico.md +180 -0
  51. equiframe/docs/compatibility.md +254 -0
  52. equiframe/docs/contributing.md +171 -0
  53. equiframe/docs/dependencies.md +237 -0
  54. equiframe/docs/eda.md +166 -0
  55. equiframe/docs/extending.md +158 -0
  56. equiframe/docs/fast-ci.md +52 -0
  57. equiframe/docs/first-use.md +78 -0
  58. equiframe/docs/index.md +142 -0
  59. equiframe/docs/ingestion.md +145 -0
  60. equiframe/docs/inicio.md +129 -0
  61. equiframe/docs/installed-private.md +97 -0
  62. equiframe/docs/journeys.md +362 -0
  63. equiframe/docs/lectura-conversion.md +211 -0
  64. equiframe/docs/licensing.md +17 -0
  65. equiframe/docs/maintenance.md +233 -0
  66. equiframe/docs/migration-ir.md +333 -0
  67. equiframe/docs/option-audit.md +45 -0
  68. equiframe/docs/outcome-gates.md +134 -0
  69. equiframe/docs/performance.md +263 -0
  70. equiframe/docs/piloto-simulado.md +56 -0
  71. equiframe/docs/quickstart.md +156 -0
  72. equiframe/docs/r_review.md +212 -0
  73. equiframe/docs/recipes.md +248 -0
  74. equiframe/docs/releasing.md +183 -0
  75. equiframe/docs/security.md +218 -0
  76. equiframe/docs/semantics.md +274 -0
  77. equiframe/docs/sqlite-snapshots.md +262 -0
  78. equiframe/docs/universal-sources.md +49 -0
  79. equiframe/docs/universalidad-cinco-pasos.md +107 -0
  80. equiframe/docs/upgrading.md +538 -0
  81. equiframe/duckdb_capture.py +410 -0
  82. equiframe/eda/__init__.py +36 -0
  83. equiframe/eda/__main__.py +3 -0
  84. equiframe/eda/_backend.py +612 -0
  85. equiframe/eda/_inventory.py +49 -0
  86. equiframe/eda/_paths.py +49 -0
  87. equiframe/eda/_report.py +291 -0
  88. equiframe/eda/api.py +535 -0
  89. equiframe/eda/cli.py +102 -0
  90. equiframe/eda/models.py +239 -0
  91. equiframe/errors.py +83 -0
  92. equiframe/evidence.py +375 -0
  93. equiframe/examples/integrations/eda_generic.py +60 -0
  94. equiframe/examples/integrations/outcome_gates.py +94 -0
  95. equiframe/examples/integrations/sqlite_snapshot.py +246 -0
  96. equiframe/examples/integrations/sqlite_snapshot_demo.py +226 -0
  97. equiframe/examples/integrations/universal_first_use.py +296 -0
  98. equiframe/examples/journeys/README.md +31 -0
  99. equiframe/examples/journeys/__init__.py +1 -0
  100. equiframe/examples/journeys/a_pandas_refactor.py +88 -0
  101. equiframe/examples/journeys/b_native_migration.py +181 -0
  102. equiframe/examples/journeys/c_retail_etl.py +460 -0
  103. equiframe/examples/journeys/common.py +136 -0
  104. equiframe/examples/journeys/d_financial_reconciliation.py +96 -0
  105. equiframe/examples/journeys/e_telemetry.py +135 -0
  106. equiframe/examples/journeys/f_data_science_preparation.py +84 -0
  107. equiframe/examples/journeys/g_external_snapshots.py +197 -0
  108. equiframe/examples/journeys/h_portable_ci.py +129 -0
  109. equiframe/examples/journeys/requirements.lock +249 -0
  110. equiframe/examples/journeys/verify_journeys.py +117 -0
  111. equiframe/examples/load_convert_compare.py +85 -0
  112. equiframe/execution.py +355 -0
  113. equiframe/execution_types.py +177 -0
  114. equiframe/ingestion.py +744 -0
  115. equiframe/local.py +126 -0
  116. equiframe/migration/__init__.py +94 -0
  117. equiframe/migration/_physical.py +8 -0
  118. equiframe/migration/backends/_timestamp_transport.py +109 -0
  119. equiframe/migration/backends/duckdb.py +651 -0
  120. equiframe/migration/backends/duckdb_decimal.py +125 -0
  121. equiframe/migration/backends/duckdb_time.py +87 -0
  122. equiframe/migration/backends/pandas.py +544 -0
  123. equiframe/migration/backends/polars.py +915 -0
  124. equiframe/migration/backends/polars_decimal.py +231 -0
  125. equiframe/migration/io.py +198 -0
  126. equiframe/migration/ports.py +96 -0
  127. equiframe/migration/runner.py +545 -0
  128. equiframe/migration/runner_types.py +405 -0
  129. equiframe/migration/types.py +354 -0
  130. equiframe/migration/validation.py +619 -0
  131. equiframe/normalization.py +358 -0
  132. equiframe/pipeline.py +274 -0
  133. equiframe/pipeline_types.py +247 -0
  134. equiframe/ports.py +39 -0
  135. equiframe/preprocessing.py +110 -0
  136. equiframe/profile_types.py +343 -0
  137. equiframe/profiles.py +447 -0
  138. equiframe/py.typed +0 -0
  139. equiframe/r_review.py +706 -0
  140. equiframe/reduction.py +518 -0
  141. equiframe/registry.py +121 -0
  142. equiframe/report_rendering.py +366 -0
  143. equiframe/report_types.py +294 -0
  144. equiframe/reporting.py +26 -0
  145. equiframe/rule_types.py +178 -0
  146. equiframe/rules.py +819 -0
  147. equiframe/schemas/r-execution-binding-1.0.schema.json +46 -0
  148. equiframe/schemas/r-review-binding-1.0.schema.json +28 -0
  149. equiframe/schemas/r-review-options-1.0.schema.json +38 -0
  150. equiframe/schemas/r-review-result-1.0.schema.json +207 -0
  151. equiframe/schemas/reviewed-decision-1.0.schema.json +3394 -0
  152. equiframe/serialization.py +152 -0
  153. equiframe/snapshot.py +296 -0
  154. equiframe/sources.py +283 -0
  155. equiframe/tables.py +459 -0
  156. equiframe/types.py +71 -0
  157. equiframe/values.py +252 -0
  158. equiframe/verification_request.py +35 -0
  159. equiframe/verify.py +799 -0
  160. equiframe/verify_types.py +255 -0
  161. equiframe/workflow.py +510 -0
  162. equiframe/workflow_request.py +38 -0
  163. equiframe-0.12.4.dist-info/METADATA +268 -0
  164. equiframe-0.12.4.dist-info/RECORD +170 -0
  165. equiframe-0.12.4.dist-info/WHEEL +4 -0
  166. equiframe-0.12.4.dist-info/entry_points.txt +3 -0
  167. equiframe-0.12.4.dist-info/licenses/LICENSE +21 -0
  168. equiframe-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md +42 -0
  169. equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/LICENSE +201 -0
  170. equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/PROVENANCE.md +18 -0
@@ -0,0 +1,612 @@
1
+ """DuckDB aggregates over one private fixed snapshot; identifiers are quoted."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Callable
7
+ from decimal import Decimal, localcontext
8
+ from fractions import Fraction
9
+ from typing import TYPE_CHECKING, Any
10
+
11
+ from equiframe._duckdb_transport import duckdb_parameters
12
+ from equiframe.errors import CapabilityError
13
+
14
+ from .models import ColumnProfile, DuplicateProfile, EdaConfig, Finding, KeyRule, NumericRule
15
+
16
+ if TYPE_CHECKING:
17
+ import duckdb
18
+
19
+
20
+ def quote(name: str) -> str:
21
+ return '"' + name.replace('"', '""') + '"'
22
+
23
+
24
+ def value(item: Any) -> Any:
25
+ """JSON transport never silently turns Decimal/nonfinite into JSON floats."""
26
+ if isinstance(item, Decimal):
27
+ return str(item)
28
+ if isinstance(item, float) and not math.isfinite(item):
29
+ return {"kind": "nan" if math.isnan(item) else "infinity", "sign": str(item)}
30
+ if hasattr(item, "isoformat"):
31
+ return item.isoformat()
32
+ return item
33
+
34
+
35
+ class Backend:
36
+ def __init__(
37
+ self,
38
+ connection: duckdb.DuckDBPyConnection,
39
+ config: EdaConfig,
40
+ checkpoint: Callable[[], None],
41
+ ) -> None:
42
+ self.connection, self.config, self.checkpoint = connection, config, checkpoint
43
+ self.findings: list[Finding] = []
44
+ self.raw_schema: dict[str, str] = {}
45
+ self.schema: dict[str, str] = {}
46
+
47
+ def query(self, sql: str, params: list[Any] | None = None) -> list[Any]:
48
+ self.checkpoint()
49
+ provided = params or []
50
+ bound = duckdb_parameters({str(i): p for i, p in enumerate(provided)})
51
+ rows = self.connection.execute(
52
+ sql, [bound[str(i)] for i in range(len(provided))]
53
+ ).fetchall()
54
+ self.checkpoint()
55
+ return rows
56
+
57
+ def examples(
58
+ self, predicate: str, columns: tuple[str, ...], params: list[Any] | None = None
59
+ ) -> tuple[dict[str, object], ...]:
60
+ names = ("__ef_eda_row", *columns)
61
+ rows = self.query(
62
+ f"SELECT {','.join(quote(c) for c in names)} FROM typed "
63
+ f"WHERE {predicate} ORDER BY __ef_eda_row LIMIT "
64
+ f"{self.config.resources.examples}",
65
+ params,
66
+ )
67
+ return tuple({name: value(v) for name, v in zip(names, row, strict=True)} for row in rows)
68
+
69
+ def columns(self, rows: int) -> tuple[ColumnProfile, ...]:
70
+ self.raw_schema = {
71
+ r[0]: r[1] for r in self.query("DESCRIBE SELECT * EXCLUDE (__ef_eda_row) FROM raw")
72
+ }
73
+ profiles = []
74
+ for name, dtype in self.raw_schema.items():
75
+ col = quote(name)
76
+ floating = dtype in ("DOUBLE", "FLOAT")
77
+ numeric = floating or dtype.startswith(
78
+ (
79
+ "DECIMAL",
80
+ "INTEGER",
81
+ "BIGINT",
82
+ "SMALLINT",
83
+ "TINYINT",
84
+ "HUGEINT",
85
+ "UBIGINT",
86
+ "UINTEGER",
87
+ "UTINYINT",
88
+ "USMALLINT",
89
+ )
90
+ )
91
+ text = dtype == "VARCHAR"
92
+ nan = f"isnan({col})" if floating else "FALSE"
93
+ nonfinite = f"NOT isfinite({col})" if floating else "FALSE"
94
+ empty = f"{col} = ''" if text else "FALSE"
95
+ zero = f"{col} = 0" if numeric else "FALSE"
96
+ present = f"{col} IS NOT NULL AND NOT ({nan})"
97
+ valid = f"{present} AND NOT ({empty}) AND NOT ({nonfinite})"
98
+ counts = self.query(
99
+ f"SELECT count(*) FILTER (WHERE {col} IS NULL), "
100
+ f"count(*) FILTER (WHERE {nan}), count(*) FILTER (WHERE {empty}), "
101
+ f"count(*) FILTER (WHERE {zero}), count(*) FILTER (WHERE {nonfinite}), "
102
+ f"count(*) FILTER (WHERE {valid}), count(DISTINCT {col}) "
103
+ f"FILTER (WHERE {present}) FROM raw"
104
+ )[0]
105
+ freq = self.query(
106
+ f"SELECT {col}, count(*) __ef_count FROM raw GROUP BY {col} "
107
+ f"ORDER BY __ef_count DESC, {col} ASC NULLS LAST LIMIT "
108
+ f"{self.config.resources.top_categories}"
109
+ )
110
+ profiles.append(
111
+ ColumnProfile(
112
+ name=name,
113
+ physical_type=dtype,
114
+ rows=rows,
115
+ nulls=counts[0],
116
+ nan=counts[1],
117
+ empty=counts[2],
118
+ zero=counts[3],
119
+ nonfinite=counts[4],
120
+ valid=counts[5],
121
+ distinct_present=counts[6],
122
+ null_rate=counts[0] / rows if rows else None,
123
+ empty_rate=counts[2] / rows if rows else None,
124
+ nan_rate=counts[1] / rows if rows else None,
125
+ frequencies=tuple(
126
+ {"value": value(v), "count": n, "denominator": rows} for v, n in freq
127
+ ),
128
+ )
129
+ )
130
+ return tuple(profiles)
131
+
132
+ def convert(self) -> tuple[dict[str, object], ...]:
133
+ expressions = ["*"]
134
+ metadata: list[dict[str, object]] = []
135
+ for conversion in self.config.conversions:
136
+ if (
137
+ conversion.column not in self.raw_schema
138
+ or conversion.target.casefold() in {n.casefold() for n in self.raw_schema}
139
+ or conversion.target.casefold().startswith("__ef_")
140
+ ):
141
+ raise ValueError("conversion requires existing source and new derived name")
142
+ if self.raw_schema[conversion.column] != "VARCHAR":
143
+ raise ValueError("MVP explicit conversion requires a text source column")
144
+ col = quote(conversion.column)
145
+ base = f"nullif({col}, '')" if conversion.empty == "missing" else col
146
+ if conversion.kind == "decimal":
147
+ # Reject excess fractional precision instead of DuckDB's rounded cast.
148
+ pattern = (
149
+ rf"[+-]?[0-9]+(\.[0-9]{{1,{conversion.scale}}})?"
150
+ if conversion.scale
151
+ else r"[+-]?[0-9]+"
152
+ )
153
+ expression = (
154
+ f"CASE WHEN regexp_full_match({base}, '{pattern}') THEN "
155
+ f"try_cast({base} AS DECIMAL({conversion.precision},{conversion.scale})) END"
156
+ )
157
+ elif conversion.kind == "integer":
158
+ expression = (
159
+ f"CASE WHEN regexp_full_match({base}, '[+-]?[0-9]+') "
160
+ f"THEN try_cast({base} AS BIGINT) END"
161
+ )
162
+ elif conversion.kind == "date":
163
+ expression = (
164
+ f"CASE WHEN regexp_full_match({base}, '[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}') "
165
+ f"THEN try_cast({base} AS DATE) END"
166
+ )
167
+ else:
168
+ # Preserve explicit instants; never guess a zone for naive timestamps.
169
+ pattern = (
170
+ r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}"
171
+ r"(\.[0-9]{1,6})?(Z|[+-][0-9]{2}:[0-9]{2})"
172
+ )
173
+ expression = (
174
+ f"CASE WHEN regexp_full_match({base}, '{pattern}') "
175
+ f"THEN try_cast({base} AS TIMESTAMPTZ) END"
176
+ )
177
+ expressions.append(expression + " AS " + quote(conversion.target))
178
+ attempted = f"{base} IS NOT NULL"
179
+ invalid = f"({attempted}) AND ({expression}) IS NULL"
180
+ attempted_n, failures, missing = self.query(
181
+ f"SELECT count(*) FILTER (WHERE {attempted}), count(*) FILTER (WHERE {invalid}), "
182
+ f"count(*) FILTER (WHERE {base} IS NULL) FROM raw"
183
+ )[0]
184
+ ex = self.query(
185
+ f"SELECT __ef_eda_row, {col} FROM raw WHERE {invalid} "
186
+ f"ORDER BY __ef_eda_row LIMIT {self.config.resources.examples}"
187
+ )
188
+ metadata.append(
189
+ {
190
+ "column": conversion.column,
191
+ "target": conversion.target,
192
+ "policy": conversion.model_dump(mode="json"),
193
+ "attempted": attempted_n,
194
+ "invalid": failures,
195
+ "missing": missing,
196
+ "examples": [{"row": r, "original": v} for r, v in ex],
197
+ }
198
+ )
199
+ if failures:
200
+ self.findings.append(
201
+ Finding(
202
+ code="conversion_invalid",
203
+ severity="rule_violation",
204
+ column=conversion.column,
205
+ count=failures,
206
+ denominator=attempted_n,
207
+ reason=(
208
+ "Explicit conversion failed; original values retained; "
209
+ "no rounding or guessed dates."
210
+ ),
211
+ examples=tuple({"row": r, "original": v} for r, v in ex),
212
+ )
213
+ )
214
+ self.query("CREATE TEMP VIEW typed AS SELECT " + ",".join(expressions) + " FROM raw")
215
+ self.schema = {r[0]: r[1] for r in self.query("DESCRIBE typed")}
216
+ for item, conversion in zip(metadata, self.config.conversions, strict=True):
217
+ if conversion.kind in ("date", "timestamp"):
218
+ col = quote(conversion.target)
219
+ low, high = self.query(
220
+ f"SELECT cast(min({col}) AS VARCHAR), cast(max({col}) AS VARCHAR) FROM typed"
221
+ )[0]
222
+ years = self.query(
223
+ f"SELECT extract(year FROM {col}) __ef_year, count(*) __ef_count FROM typed "
224
+ f"WHERE {col} IS NOT NULL GROUP BY 1 ORDER BY 1"
225
+ )
226
+ item["observed_dates"] = {
227
+ "minimum": value(low),
228
+ "maximum": value(high),
229
+ "years": [{"year": int(y), "rows": n} for y, n in years],
230
+ "timezone": "UTC for explicit instants; dates unchanged",
231
+ "cohort_inferred": False,
232
+ }
233
+ return tuple(metadata)
234
+
235
+ def duplicates(self, columns: tuple[str, ...], rule: KeyRule | None = None) -> DuplicateProfile:
236
+ if not set(columns) <= self.raw_schema.keys():
237
+ raise ValueError("duplicate rule references unknown source column")
238
+ names = ",".join(quote(c) for c in columns)
239
+ predicates = []
240
+ for name in columns:
241
+ col, dtype = quote(name), self.raw_schema[name]
242
+ predicates.append(f"{col} IS NULL")
243
+ if dtype in ("DOUBLE", "FLOAT"):
244
+ predicates.append(f"isnan({col})")
245
+ if rule is not None and rule.empty_is_missing and dtype == "VARCHAR":
246
+ predicates.append(f"{col} = ''")
247
+ missing = " OR ".join(predicates)
248
+ incomplete = self.query(f"SELECT count(*) FROM raw WHERE {missing}")[0][0] if rule else 0
249
+ where = f"WHERE NOT ({missing})" if rule and rule.incomplete != "include" else ""
250
+ denominator = self.query(f"SELECT count(*) FROM raw {where}")[0][0]
251
+ repeated = (
252
+ f"SELECT {names}, count(*) __ef_count FROM raw {where} "
253
+ f"GROUP BY {names} HAVING count(*) > 1"
254
+ )
255
+ groups, involved, excess = self.query(
256
+ f"SELECT count(*), coalesce(sum(__ef_count),0), "
257
+ f"coalesce(sum(__ef_count-1),0) FROM ({repeated})"
258
+ )[0]
259
+ examples = self.query(
260
+ f"SELECT {names}, __ef_count FROM ({repeated}) ORDER BY __ef_count DESC, {names} "
261
+ f"LIMIT {self.config.resources.examples}"
262
+ )
263
+ # Group examples include bounded row references without loading groups in Python.
264
+ references = self.query(
265
+ f"SELECT __ef_eda_row FROM (SELECT *, count(*) OVER "
266
+ f"(PARTITION BY {names}) __ef_n FROM raw {where}) "
267
+ f"WHERE __ef_n > 1 ORDER BY __ef_eda_row LIMIT {self.config.resources.examples}"
268
+ )
269
+ if rule is not None and incomplete and rule.incomplete == "reject":
270
+ self.findings.append(
271
+ Finding(
272
+ code="incomplete_key",
273
+ severity="rule_violation",
274
+ count=incomplete,
275
+ denominator=denominator + incomplete,
276
+ reason="Declared key policy rejects incomplete keys; source retained.",
277
+ examples=self.examples(missing, columns),
278
+ )
279
+ )
280
+ if groups:
281
+ self.findings.append(
282
+ Finding(
283
+ code="key_duplicates" if rule else "row_duplicates",
284
+ severity="rule_violation" if rule and rule.require_unique else "observation",
285
+ count=int(excess),
286
+ denominator=denominator,
287
+ reason="Repetitions observed; may be legitimate history. No row was removed.",
288
+ examples=tuple({"row": r[0]} for r in references),
289
+ )
290
+ )
291
+ return DuplicateProfile(
292
+ columns=columns,
293
+ denominator=denominator,
294
+ incomplete_rows=incomplete,
295
+ policy=rule.incomplete if rule else "all_rows_including_null_and_nan",
296
+ repeated_groups=int(groups),
297
+ involved_rows=int(involved),
298
+ excess_rows=int(excess),
299
+ excess_rate=float(excess / denominator) if denominator else None,
300
+ examples=tuple(
301
+ {
302
+ "values": {c: value(v) for c, v in zip(columns, row[:-1], strict=True)},
303
+ "rows": row[-1],
304
+ }
305
+ for row in examples
306
+ ),
307
+ )
308
+
309
+ def numeric(self, rule: NumericRule) -> dict[str, object]:
310
+ if rule.column not in self.schema or not set(rule.groups) <= self.schema.keys():
311
+ raise ValueError("numeric rule references unknown column")
312
+ dtype, col = self.schema[rule.column], quote(rule.column)
313
+ if dtype not in (
314
+ "DOUBLE",
315
+ "FLOAT",
316
+ "INTEGER",
317
+ "BIGINT",
318
+ "SMALLINT",
319
+ "TINYINT",
320
+ "HUGEINT",
321
+ "UBIGINT",
322
+ "UINTEGER",
323
+ "UTINYINT",
324
+ "USMALLINT",
325
+ ) and not dtype.startswith("DECIMAL"):
326
+ raise ValueError("numeric column must be numeric or explicitly converted")
327
+ if rule.monetary and not dtype.startswith("DECIMAL"):
328
+ raise ValueError("monetary diagnostics require exact Decimal, never float")
329
+ if rule.monetary and (
330
+ rule.currency_column is None or self.schema[rule.currency_column] != "VARCHAR"
331
+ ):
332
+ raise ValueError("currency identifiers must be explicit text, not numeric groups")
333
+ if any(
334
+ self.schema[g]
335
+ not in (
336
+ "VARCHAR",
337
+ "BOOLEAN",
338
+ "INTEGER",
339
+ "BIGINT",
340
+ "FLOAT",
341
+ "DOUBLE",
342
+ "UTINYINT",
343
+ "USMALLINT",
344
+ "SMALLINT",
345
+ "TINYINT",
346
+ "UINTEGER",
347
+ "UBIGINT",
348
+ )
349
+ and not self.schema[g].startswith("DECIMAL")
350
+ for g in rule.groups
351
+ ):
352
+ raise CapabilityError("MVP comparable groups admit text/bool/integer/Decimal/float")
353
+ names = ",".join(quote(g) for g in rule.groups)
354
+ group_count = (
355
+ self.query(f"SELECT count(*) FROM (SELECT {names} FROM typed GROUP BY {names})")[0][0]
356
+ if names
357
+ else 1
358
+ )
359
+ if group_count > self.config.resources.max_groups:
360
+ self.findings.append(
361
+ Finding(
362
+ code="group_limit",
363
+ severity="not_evaluable",
364
+ column=rule.column,
365
+ reason=(
366
+ "Comparable group count exceeds configured budget; "
367
+ "no groups silently discarded."
368
+ ),
369
+ )
370
+ )
371
+ return {
372
+ "rule": rule.model_dump(mode="json"),
373
+ "state": "not_evaluable",
374
+ "reason": "group_limit",
375
+ "groups_total": group_count,
376
+ }
377
+ groups = (
378
+ self.query(f"SELECT {names} FROM typed GROUP BY {names} ORDER BY {names}")
379
+ if names
380
+ else [()]
381
+ )
382
+ output = []
383
+ for group in groups:
384
+ where = (
385
+ " AND ".join(f"{quote(c)} IS NOT DISTINCT FROM ?" for c in rule.groups) or "TRUE"
386
+ )
387
+ float_type = dtype in ("DOUBLE", "FLOAT")
388
+ finite = f"{col} IS NOT NULL" + (f" AND isfinite({col})" if float_type else "")
389
+ valid_where = f"({where}) AND ({finite})"
390
+ total, valid, missing, nonfinite = self.query(
391
+ f"SELECT count(*), count(*) FILTER (WHERE {finite}), "
392
+ f"count(*) FILTER (WHERE {col} IS NULL), "
393
+ f"count(*) FILTER (WHERE {col} IS NOT NULL AND NOT ({finite})) "
394
+ f"FROM typed WHERE {where}",
395
+ list(group),
396
+ )[0]
397
+ record: dict[str, object] = {
398
+ "group": {c: value(v) for c, v in zip(rule.groups, group, strict=True)},
399
+ "total": total,
400
+ "valid": valid,
401
+ "missing": missing,
402
+ "nonfinite": nonfinite,
403
+ "state": "completed" if valid else "not_evaluable",
404
+ "reason": None if valid else "no_finite_values",
405
+ }
406
+ incomparable = (
407
+ rule.monetary and group[rule.groups.index(rule.currency_column)] in (None, "")
408
+ ) or any(isinstance(v, float) and not math.isfinite(v) for v in group)
409
+ if incomparable:
410
+ record.update(
411
+ state="not_evaluable",
412
+ reason="missing_currency" if rule.monetary else "nonfinite_group",
413
+ outliers={
414
+ "method": rule.outliers,
415
+ "state": "not_evaluable",
416
+ "count": None,
417
+ "reason": "missing_currency" if rule.monetary else "nonfinite_group",
418
+ },
419
+ )
420
+ self.findings.append(
421
+ Finding(
422
+ code="incomparable_group",
423
+ severity="not_evaluable",
424
+ column=rule.column,
425
+ denominator=valid,
426
+ reason=(
427
+ "Grouping has missing currency or nonfinite values; "
428
+ "no pooled numeric amounts."
429
+ ),
430
+ )
431
+ )
432
+ output.append(record)
433
+ continue
434
+ if not valid:
435
+ output.append(record)
436
+ continue
437
+ percentiles = sorted(set((0.25, 0.5, 0.75, *self.config.percentiles)))
438
+ qs = ",".join(f"quantile_disc({col}, {p})" for p in percentiles)
439
+ sums = self.query(
440
+ f"SELECT min({col}), max({col}), sum({col}), avg({col}), "
441
+ f"stddev_{'pop' if self.config.stddev == 'population' else 'samp'}({col}), {qs} "
442
+ f"FROM typed WHERE {valid_where}",
443
+ list(group),
444
+ )[0]
445
+ qmap = dict(zip(percentiles, sums[5:], strict=True))
446
+ record.update(
447
+ minimum=value(sums[0]),
448
+ maximum=value(sums[1]),
449
+ sum=value(sums[2]),
450
+ mean_approximate=value(sums[3]),
451
+ stddev_approximate=value(sums[4]),
452
+ stddev_definition=self.config.stddev,
453
+ quantiles={str(p): value(qmap[p]) for p in percentiles},
454
+ median=value(qmap[0.5]),
455
+ quantile_method=(
456
+ "DuckDB quantile_disc, Hyndman-Fan type 1; observed value, no interpolation"
457
+ ),
458
+ precision=(
459
+ "Exact sum/min/max/discrete quantiles for integer/Decimal; "
460
+ "mean/stddev float64 approximate; float aggregates approximate"
461
+ ),
462
+ excluded_from_statistics=(
463
+ "NULL and nonfinite; failed conversions remain separate in conversion receipt"
464
+ ),
465
+ )
466
+ if isinstance(sums[2], (int, Decimal)):
467
+ mean = Fraction(sums[2]) / valid
468
+ record["mean_exact_rational"] = {
469
+ "numerator": str(mean.numerator),
470
+ "denominator": str(mean.denominator),
471
+ }
472
+ outlier: dict[str, object] = {
473
+ "method": rule.outliers,
474
+ "multiplier": rule.multiplier,
475
+ "min_support": rule.min_support,
476
+ "state": "not_evaluable",
477
+ "count": None,
478
+ "reason": "disabled" if rule.outliers == "none" else "insufficient_support",
479
+ }
480
+ if valid >= rule.min_support and rule.outliers != "none":
481
+ median = qmap[0.5]
482
+ with localcontext() as ctx:
483
+ ctx.prec = 80
484
+ if rule.outliers == "iqr":
485
+ dispersion = Decimal(str(qmap[0.75])) - Decimal(str(qmap[0.25]))
486
+ lower = (
487
+ Decimal(str(qmap[0.25])) - Decimal(str(rule.multiplier)) * dispersion
488
+ )
489
+ upper = (
490
+ Decimal(str(qmap[0.75])) + Decimal(str(rule.multiplier)) * dispersion
491
+ )
492
+ else:
493
+ mad = self.query(
494
+ f"SELECT quantile_disc(abs({col} - ?),0.5) "
495
+ f"FROM typed WHERE {valid_where}",
496
+ [median, *group],
497
+ )[0][0]
498
+ dispersion = Decimal(str(mad))
499
+ lower = Decimal(str(median)) - Decimal(str(rule.multiplier)) * dispersion
500
+ upper = Decimal(str(median)) + Decimal(str(rule.multiplier)) * dispersion
501
+ if dispersion == 0:
502
+ outlier["reason"] = "zero_dispersion"
503
+ else:
504
+ predicate = f"{valid_where} AND ({col} < ? OR {col} > ?)"
505
+ # Reduce only transient bounds by exact trailing-zero removal,
506
+ # within the high-precision local context, never original money.
507
+ lower = lower.normalize()
508
+ upper = upper.normalize()
509
+ if int(lower.as_tuple().exponent) > 0:
510
+ lower = Decimal(format(lower, "f"))
511
+ if int(upper.as_tuple().exponent) > 0:
512
+ upper = Decimal(format(upper, "f"))
513
+ params = [*group, lower, upper]
514
+ try:
515
+ count = self.query(
516
+ f"SELECT count(*) FROM typed WHERE {predicate}", params
517
+ )[0][0]
518
+ ex = self.examples(predicate, (rule.column, *rule.groups), params)
519
+ except CapabilityError:
520
+ outlier["reason"] = "unrepresentable_decimal_bounds"
521
+ record["outliers"] = outlier
522
+ self.findings.append(
523
+ Finding(
524
+ code="outliers_unavailable",
525
+ severity="not_evaluable",
526
+ column=rule.column,
527
+ denominator=valid,
528
+ reason=(
529
+ "Exact outlier bounds cannot be admitted by DuckDB; "
530
+ "no float fallback."
531
+ ),
532
+ )
533
+ )
534
+ output.append(record)
535
+ continue
536
+ outlier.update(
537
+ state="completed",
538
+ reason=None,
539
+ count=count,
540
+ lower=str(lower),
541
+ upper=str(upper),
542
+ dispersion=str(dispersion),
543
+ examples=list(ex),
544
+ denominator=valid,
545
+ interpretation=(
546
+ "Case for investigation; not confirmed error; unscaled MAD or IQR"
547
+ ),
548
+ )
549
+ if count:
550
+ self.findings.append(
551
+ Finding(
552
+ code="outlier_candidate",
553
+ severity="observation",
554
+ column=rule.column,
555
+ count=count,
556
+ denominator=valid,
557
+ reason=(
558
+ "Outside configured bounds in a comparable group; "
559
+ "legitimate extremes remain intact."
560
+ ),
561
+ examples=ex,
562
+ )
563
+ )
564
+ record["outliers"] = outlier
565
+ if outlier["state"] == "not_evaluable" and rule.outliers != "none":
566
+ self.findings.append(
567
+ Finding(
568
+ code="outliers_unavailable",
569
+ severity="not_evaluable",
570
+ column=rule.column,
571
+ denominator=valid,
572
+ reason=str(outlier["reason"]),
573
+ )
574
+ )
575
+ if not incomparable:
576
+ # Histogram is full-population; bounds/bins use declared float visualization only.
577
+ hist = self.query(
578
+ f"SELECT b, count(*) FROM (SELECT CASE WHEN ? = ? THEN 0 ELSE "
579
+ f"least({self.config.resources.histogram_bins - 1}, greatest(0, "
580
+ f"floor((cast({col} AS DOUBLE)-?) / (? - ?) * "
581
+ f"{self.config.resources.histogram_bins}))) END b FROM typed "
582
+ f"WHERE {valid_where}) GROUP BY b ORDER BY b",
583
+ [
584
+ float(sums[0]),
585
+ float(sums[1]),
586
+ float(sums[0]),
587
+ float(sums[1]),
588
+ float(sums[0]),
589
+ *group,
590
+ ],
591
+ )
592
+ record["histogram"] = [{"bin": int(b), "count": n} for b, n in hist]
593
+ record["histogram_bounds_approximate"] = [float(sums[0]), float(sums[1])]
594
+ sample = self.query(
595
+ f"SELECT __ef_eda_row, {col} FROM typed WHERE {valid_where} "
596
+ f"ORDER BY __ef_eda_row LIMIT {self.config.resources.chart_rows}",
597
+ list(group),
598
+ )
599
+ record["chart_sample"] = [{"row": r, "value": value(v)} for r, v in sample]
600
+ record["chart_scope"] = (
601
+ "Deterministic first finite rows in source order; visualization only, "
602
+ "not representative; all statistics calculated on full group"
603
+ )
604
+ output.append(record)
605
+ return {
606
+ "rule": rule.model_dump(mode="json"),
607
+ "groups_total": group_count,
608
+ "groups": output,
609
+ "state": "not_evaluable"
610
+ if any(r["state"] != "completed" for r in output)
611
+ else "completed",
612
+ }
@@ -0,0 +1,49 @@
1
+ """Explicit first EDA public surface; update only with reviewed contract changes."""
2
+
3
+ INVENTORY = {
4
+ "status": "experimental",
5
+ "format_version": "1.0",
6
+ "public_names": (
7
+ "ColumnProfile",
8
+ "Conversion",
9
+ "DuplicateProfile",
10
+ "EdaConfig",
11
+ "EdaManifest",
12
+ "EdaResult",
13
+ "Finding",
14
+ "KeyRule",
15
+ "NumericRule",
16
+ "Resources",
17
+ "ResultFile",
18
+ "Source",
19
+ "analyze",
20
+ "analyze_artifact",
21
+ "analyze_migration",
22
+ "read_result",
23
+ ),
24
+ "signatures": {
25
+ "analyze": "(source: 'Source', *, config: 'EdaConfig | None' = None, "
26
+ "output: 'Path', cancellation: 'Callable[[], bool] | None' = "
27
+ "None) -> 'EdaResult'",
28
+ "analyze_artifact": "(artifact: 'TableArtifact', *, config: 'EdaConfig | "
29
+ "None' = None, output: 'Path') -> 'EdaResult'",
30
+ "analyze_migration": "(path: 'Path', *, side: \"Literal['source', "
31
+ "'target']\" = 'source', config: 'EdaConfig | "
32
+ "None' = None, output: 'Path') -> 'EdaResult'",
33
+ "read_result": "(path: 'Path') -> 'EdaResult'",
34
+ },
35
+ "model_shapes": {
36
+ "ColumnProfile": "fef8bbba53cfe502ec13bdbdb6c20e2635a1c9865b92655cebe367ace0ce81e9",
37
+ "Conversion": "f6f432c6c02a2f3f770d6fc602cf6dc6202302b46bdf670540cc2f2dadc00f95",
38
+ "DuplicateProfile": "2a1f6d9efd1618d82de32b4c4d5948f3a0d89b0452924da3c495c69caa9eeaa6",
39
+ "EdaConfig": "65ea2de1dd1d43bd0f1bb8942dd5b28ff7a95f346c24edc01418ec5a04083fe1",
40
+ "EdaManifest": "f9d3b6ccb63249f7bd796f8154135a04c844578231666a988e400c857d0b3e2d",
41
+ "EdaResult": "5bce9ab1457ab99b5ed818ecaac7695caef205c12eff29dd0fe6507e8c8a0cad",
42
+ "Finding": "331db5cf15d9e26ef8b94e2fd25e625e2e804165f6c584d78a6a3c83ea41faa2",
43
+ "KeyRule": "4b0f8f2f6baf3eca22ab5c0da607d90f46c61bec008b5d8a760b61372a18828e",
44
+ "NumericRule": "16951b2437617d4d52437d3124d7b394f546ba49eac98f9c049ab07fefe21938",
45
+ "Resources": "82173ef020b2937c4c7aaf1cea037effbc836dfecf25e5929c29cd3702898e9a",
46
+ "ResultFile": "eb3f6aa3e82b85493541404add0212327cf0b6223f2c858cd615d83df1e1b7e8",
47
+ "Source": "182d1cc45e149e0251acdae721dd78b4f73721d69cd52f630140c805b96265fb",
48
+ },
49
+ }