equiframe 0.12.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- equiframe/__init__.py +832 -0
- equiframe/__main__.py +3 -0
- equiframe/_base.py +84 -0
- equiframe/_cli_support.py +125 -0
- equiframe/_comparison_io.py +161 -0
- equiframe/_dataset_provenance.py +146 -0
- equiframe/_decimal.py +22 -0
- equiframe/_diff_capture.py +433 -0
- equiframe/_doctor.py +123 -0
- equiframe/_document_io.py +61 -0
- equiframe/_documents.py +79 -0
- equiframe/_duckdb_transport.py +149 -0
- equiframe/_error_reporting.py +52 -0
- equiframe/_first_use.py +102 -0
- equiframe/_path_compat.py +26 -0
- equiframe/_physical.py +133 -0
- equiframe/_r_review.R +308 -0
- equiframe/_regular_io.py +167 -0
- equiframe/_report_format_v1.py +298 -0
- equiframe/_report_io.py +591 -0
- equiframe/_stability_inventory.py +1421 -0
- equiframe/_timezones.py +257 -0
- equiframe/_vendor/cyclonedx/LICENSE +201 -0
- equiframe/_vendor/cyclonedx/PROVENANCE.md +18 -0
- equiframe/_vendor/cyclonedx/bom-1.6.schema.json +5673 -0
- equiframe/_vendor/cyclonedx/jsf-0.82.schema.json +240 -0
- equiframe/_vendor/cyclonedx/spdx.schema.json +712 -0
- equiframe/adapters.py +454 -0
- equiframe/assertion_types.py +122 -0
- equiframe/assertions.py +271 -0
- equiframe/benchmark.py +101 -0
- equiframe/bundle.py +240 -0
- equiframe/cli.py +693 -0
- equiframe/comparison.py +937 -0
- equiframe/comparison_report.py +660 -0
- equiframe/comparison_types.py +257 -0
- equiframe/compatibility.py +546 -0
- equiframe/contracts.py +879 -0
- equiframe/datasets.py +1301 -0
- equiframe/decision.py +34 -0
- equiframe/destinations.py +295 -0
- equiframe/detector_types.py +431 -0
- equiframe/detectors.py +389 -0
- equiframe/diff.py +192 -0
- equiframe/diff_contracts.py +523 -0
- equiframe/docs/adoption.md +329 -0
- equiframe/docs/api.md +534 -0
- equiframe/docs/arquitecto.md +143 -0
- equiframe/docs/choosing-api.md +84 -0
- equiframe/docs/cientifico.md +180 -0
- equiframe/docs/compatibility.md +254 -0
- equiframe/docs/contributing.md +171 -0
- equiframe/docs/dependencies.md +237 -0
- equiframe/docs/eda.md +166 -0
- equiframe/docs/extending.md +158 -0
- equiframe/docs/fast-ci.md +52 -0
- equiframe/docs/first-use.md +78 -0
- equiframe/docs/index.md +142 -0
- equiframe/docs/ingestion.md +145 -0
- equiframe/docs/inicio.md +129 -0
- equiframe/docs/installed-private.md +97 -0
- equiframe/docs/journeys.md +362 -0
- equiframe/docs/lectura-conversion.md +211 -0
- equiframe/docs/licensing.md +17 -0
- equiframe/docs/maintenance.md +233 -0
- equiframe/docs/migration-ir.md +333 -0
- equiframe/docs/option-audit.md +45 -0
- equiframe/docs/outcome-gates.md +134 -0
- equiframe/docs/performance.md +263 -0
- equiframe/docs/piloto-simulado.md +56 -0
- equiframe/docs/quickstart.md +156 -0
- equiframe/docs/r_review.md +212 -0
- equiframe/docs/recipes.md +248 -0
- equiframe/docs/releasing.md +183 -0
- equiframe/docs/security.md +218 -0
- equiframe/docs/semantics.md +274 -0
- equiframe/docs/sqlite-snapshots.md +262 -0
- equiframe/docs/universal-sources.md +49 -0
- equiframe/docs/universalidad-cinco-pasos.md +107 -0
- equiframe/docs/upgrading.md +538 -0
- equiframe/duckdb_capture.py +410 -0
- equiframe/eda/__init__.py +36 -0
- equiframe/eda/__main__.py +3 -0
- equiframe/eda/_backend.py +612 -0
- equiframe/eda/_inventory.py +49 -0
- equiframe/eda/_paths.py +49 -0
- equiframe/eda/_report.py +291 -0
- equiframe/eda/api.py +535 -0
- equiframe/eda/cli.py +102 -0
- equiframe/eda/models.py +239 -0
- equiframe/errors.py +83 -0
- equiframe/evidence.py +375 -0
- equiframe/examples/integrations/eda_generic.py +60 -0
- equiframe/examples/integrations/outcome_gates.py +94 -0
- equiframe/examples/integrations/sqlite_snapshot.py +246 -0
- equiframe/examples/integrations/sqlite_snapshot_demo.py +226 -0
- equiframe/examples/integrations/universal_first_use.py +296 -0
- equiframe/examples/journeys/README.md +31 -0
- equiframe/examples/journeys/__init__.py +1 -0
- equiframe/examples/journeys/a_pandas_refactor.py +88 -0
- equiframe/examples/journeys/b_native_migration.py +181 -0
- equiframe/examples/journeys/c_retail_etl.py +460 -0
- equiframe/examples/journeys/common.py +136 -0
- equiframe/examples/journeys/d_financial_reconciliation.py +96 -0
- equiframe/examples/journeys/e_telemetry.py +135 -0
- equiframe/examples/journeys/f_data_science_preparation.py +84 -0
- equiframe/examples/journeys/g_external_snapshots.py +197 -0
- equiframe/examples/journeys/h_portable_ci.py +129 -0
- equiframe/examples/journeys/requirements.lock +249 -0
- equiframe/examples/journeys/verify_journeys.py +117 -0
- equiframe/examples/load_convert_compare.py +85 -0
- equiframe/execution.py +355 -0
- equiframe/execution_types.py +177 -0
- equiframe/ingestion.py +744 -0
- equiframe/local.py +126 -0
- equiframe/migration/__init__.py +94 -0
- equiframe/migration/_physical.py +8 -0
- equiframe/migration/backends/_timestamp_transport.py +109 -0
- equiframe/migration/backends/duckdb.py +651 -0
- equiframe/migration/backends/duckdb_decimal.py +125 -0
- equiframe/migration/backends/duckdb_time.py +87 -0
- equiframe/migration/backends/pandas.py +544 -0
- equiframe/migration/backends/polars.py +915 -0
- equiframe/migration/backends/polars_decimal.py +231 -0
- equiframe/migration/io.py +198 -0
- equiframe/migration/ports.py +96 -0
- equiframe/migration/runner.py +545 -0
- equiframe/migration/runner_types.py +405 -0
- equiframe/migration/types.py +354 -0
- equiframe/migration/validation.py +619 -0
- equiframe/normalization.py +358 -0
- equiframe/pipeline.py +274 -0
- equiframe/pipeline_types.py +247 -0
- equiframe/ports.py +39 -0
- equiframe/preprocessing.py +110 -0
- equiframe/profile_types.py +343 -0
- equiframe/profiles.py +447 -0
- equiframe/py.typed +0 -0
- equiframe/r_review.py +706 -0
- equiframe/reduction.py +518 -0
- equiframe/registry.py +121 -0
- equiframe/report_rendering.py +366 -0
- equiframe/report_types.py +294 -0
- equiframe/reporting.py +26 -0
- equiframe/rule_types.py +178 -0
- equiframe/rules.py +819 -0
- equiframe/schemas/r-execution-binding-1.0.schema.json +46 -0
- equiframe/schemas/r-review-binding-1.0.schema.json +28 -0
- equiframe/schemas/r-review-options-1.0.schema.json +38 -0
- equiframe/schemas/r-review-result-1.0.schema.json +207 -0
- equiframe/schemas/reviewed-decision-1.0.schema.json +3394 -0
- equiframe/serialization.py +152 -0
- equiframe/snapshot.py +296 -0
- equiframe/sources.py +283 -0
- equiframe/tables.py +459 -0
- equiframe/types.py +71 -0
- equiframe/values.py +252 -0
- equiframe/verification_request.py +35 -0
- equiframe/verify.py +799 -0
- equiframe/verify_types.py +255 -0
- equiframe/workflow.py +510 -0
- equiframe/workflow_request.py +38 -0
- equiframe-0.12.4.dist-info/METADATA +268 -0
- equiframe-0.12.4.dist-info/RECORD +170 -0
- equiframe-0.12.4.dist-info/WHEEL +4 -0
- equiframe-0.12.4.dist-info/entry_points.txt +3 -0
- equiframe-0.12.4.dist-info/licenses/LICENSE +21 -0
- equiframe-0.12.4.dist-info/licenses/THIRD_PARTY_NOTICES.md +42 -0
- equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/LICENSE +201 -0
- equiframe-0.12.4.dist-info/licenses/tools/cyclonedx/PROVENANCE.md +18 -0
|
@@ -0,0 +1,612 @@
|
|
|
1
|
+
"""DuckDB aggregates over one private fixed snapshot; identifiers are quoted."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from decimal import Decimal, localcontext
|
|
8
|
+
from fractions import Fraction
|
|
9
|
+
from typing import TYPE_CHECKING, Any
|
|
10
|
+
|
|
11
|
+
from equiframe._duckdb_transport import duckdb_parameters
|
|
12
|
+
from equiframe.errors import CapabilityError
|
|
13
|
+
|
|
14
|
+
from .models import ColumnProfile, DuplicateProfile, EdaConfig, Finding, KeyRule, NumericRule
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
import duckdb
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def quote(name: str) -> str:
|
|
21
|
+
return '"' + name.replace('"', '""') + '"'
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def value(item: Any) -> Any:
|
|
25
|
+
"""JSON transport never silently turns Decimal/nonfinite into JSON floats."""
|
|
26
|
+
if isinstance(item, Decimal):
|
|
27
|
+
return str(item)
|
|
28
|
+
if isinstance(item, float) and not math.isfinite(item):
|
|
29
|
+
return {"kind": "nan" if math.isnan(item) else "infinity", "sign": str(item)}
|
|
30
|
+
if hasattr(item, "isoformat"):
|
|
31
|
+
return item.isoformat()
|
|
32
|
+
return item
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Backend:
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
connection: duckdb.DuckDBPyConnection,
|
|
39
|
+
config: EdaConfig,
|
|
40
|
+
checkpoint: Callable[[], None],
|
|
41
|
+
) -> None:
|
|
42
|
+
self.connection, self.config, self.checkpoint = connection, config, checkpoint
|
|
43
|
+
self.findings: list[Finding] = []
|
|
44
|
+
self.raw_schema: dict[str, str] = {}
|
|
45
|
+
self.schema: dict[str, str] = {}
|
|
46
|
+
|
|
47
|
+
def query(self, sql: str, params: list[Any] | None = None) -> list[Any]:
|
|
48
|
+
self.checkpoint()
|
|
49
|
+
provided = params or []
|
|
50
|
+
bound = duckdb_parameters({str(i): p for i, p in enumerate(provided)})
|
|
51
|
+
rows = self.connection.execute(
|
|
52
|
+
sql, [bound[str(i)] for i in range(len(provided))]
|
|
53
|
+
).fetchall()
|
|
54
|
+
self.checkpoint()
|
|
55
|
+
return rows
|
|
56
|
+
|
|
57
|
+
def examples(
|
|
58
|
+
self, predicate: str, columns: tuple[str, ...], params: list[Any] | None = None
|
|
59
|
+
) -> tuple[dict[str, object], ...]:
|
|
60
|
+
names = ("__ef_eda_row", *columns)
|
|
61
|
+
rows = self.query(
|
|
62
|
+
f"SELECT {','.join(quote(c) for c in names)} FROM typed "
|
|
63
|
+
f"WHERE {predicate} ORDER BY __ef_eda_row LIMIT "
|
|
64
|
+
f"{self.config.resources.examples}",
|
|
65
|
+
params,
|
|
66
|
+
)
|
|
67
|
+
return tuple({name: value(v) for name, v in zip(names, row, strict=True)} for row in rows)
|
|
68
|
+
|
|
69
|
+
def columns(self, rows: int) -> tuple[ColumnProfile, ...]:
|
|
70
|
+
self.raw_schema = {
|
|
71
|
+
r[0]: r[1] for r in self.query("DESCRIBE SELECT * EXCLUDE (__ef_eda_row) FROM raw")
|
|
72
|
+
}
|
|
73
|
+
profiles = []
|
|
74
|
+
for name, dtype in self.raw_schema.items():
|
|
75
|
+
col = quote(name)
|
|
76
|
+
floating = dtype in ("DOUBLE", "FLOAT")
|
|
77
|
+
numeric = floating or dtype.startswith(
|
|
78
|
+
(
|
|
79
|
+
"DECIMAL",
|
|
80
|
+
"INTEGER",
|
|
81
|
+
"BIGINT",
|
|
82
|
+
"SMALLINT",
|
|
83
|
+
"TINYINT",
|
|
84
|
+
"HUGEINT",
|
|
85
|
+
"UBIGINT",
|
|
86
|
+
"UINTEGER",
|
|
87
|
+
"UTINYINT",
|
|
88
|
+
"USMALLINT",
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
text = dtype == "VARCHAR"
|
|
92
|
+
nan = f"isnan({col})" if floating else "FALSE"
|
|
93
|
+
nonfinite = f"NOT isfinite({col})" if floating else "FALSE"
|
|
94
|
+
empty = f"{col} = ''" if text else "FALSE"
|
|
95
|
+
zero = f"{col} = 0" if numeric else "FALSE"
|
|
96
|
+
present = f"{col} IS NOT NULL AND NOT ({nan})"
|
|
97
|
+
valid = f"{present} AND NOT ({empty}) AND NOT ({nonfinite})"
|
|
98
|
+
counts = self.query(
|
|
99
|
+
f"SELECT count(*) FILTER (WHERE {col} IS NULL), "
|
|
100
|
+
f"count(*) FILTER (WHERE {nan}), count(*) FILTER (WHERE {empty}), "
|
|
101
|
+
f"count(*) FILTER (WHERE {zero}), count(*) FILTER (WHERE {nonfinite}), "
|
|
102
|
+
f"count(*) FILTER (WHERE {valid}), count(DISTINCT {col}) "
|
|
103
|
+
f"FILTER (WHERE {present}) FROM raw"
|
|
104
|
+
)[0]
|
|
105
|
+
freq = self.query(
|
|
106
|
+
f"SELECT {col}, count(*) __ef_count FROM raw GROUP BY {col} "
|
|
107
|
+
f"ORDER BY __ef_count DESC, {col} ASC NULLS LAST LIMIT "
|
|
108
|
+
f"{self.config.resources.top_categories}"
|
|
109
|
+
)
|
|
110
|
+
profiles.append(
|
|
111
|
+
ColumnProfile(
|
|
112
|
+
name=name,
|
|
113
|
+
physical_type=dtype,
|
|
114
|
+
rows=rows,
|
|
115
|
+
nulls=counts[0],
|
|
116
|
+
nan=counts[1],
|
|
117
|
+
empty=counts[2],
|
|
118
|
+
zero=counts[3],
|
|
119
|
+
nonfinite=counts[4],
|
|
120
|
+
valid=counts[5],
|
|
121
|
+
distinct_present=counts[6],
|
|
122
|
+
null_rate=counts[0] / rows if rows else None,
|
|
123
|
+
empty_rate=counts[2] / rows if rows else None,
|
|
124
|
+
nan_rate=counts[1] / rows if rows else None,
|
|
125
|
+
frequencies=tuple(
|
|
126
|
+
{"value": value(v), "count": n, "denominator": rows} for v, n in freq
|
|
127
|
+
),
|
|
128
|
+
)
|
|
129
|
+
)
|
|
130
|
+
return tuple(profiles)
|
|
131
|
+
|
|
132
|
+
def convert(self) -> tuple[dict[str, object], ...]:
|
|
133
|
+
expressions = ["*"]
|
|
134
|
+
metadata: list[dict[str, object]] = []
|
|
135
|
+
for conversion in self.config.conversions:
|
|
136
|
+
if (
|
|
137
|
+
conversion.column not in self.raw_schema
|
|
138
|
+
or conversion.target.casefold() in {n.casefold() for n in self.raw_schema}
|
|
139
|
+
or conversion.target.casefold().startswith("__ef_")
|
|
140
|
+
):
|
|
141
|
+
raise ValueError("conversion requires existing source and new derived name")
|
|
142
|
+
if self.raw_schema[conversion.column] != "VARCHAR":
|
|
143
|
+
raise ValueError("MVP explicit conversion requires a text source column")
|
|
144
|
+
col = quote(conversion.column)
|
|
145
|
+
base = f"nullif({col}, '')" if conversion.empty == "missing" else col
|
|
146
|
+
if conversion.kind == "decimal":
|
|
147
|
+
# Reject excess fractional precision instead of DuckDB's rounded cast.
|
|
148
|
+
pattern = (
|
|
149
|
+
rf"[+-]?[0-9]+(\.[0-9]{{1,{conversion.scale}}})?"
|
|
150
|
+
if conversion.scale
|
|
151
|
+
else r"[+-]?[0-9]+"
|
|
152
|
+
)
|
|
153
|
+
expression = (
|
|
154
|
+
f"CASE WHEN regexp_full_match({base}, '{pattern}') THEN "
|
|
155
|
+
f"try_cast({base} AS DECIMAL({conversion.precision},{conversion.scale})) END"
|
|
156
|
+
)
|
|
157
|
+
elif conversion.kind == "integer":
|
|
158
|
+
expression = (
|
|
159
|
+
f"CASE WHEN regexp_full_match({base}, '[+-]?[0-9]+') "
|
|
160
|
+
f"THEN try_cast({base} AS BIGINT) END"
|
|
161
|
+
)
|
|
162
|
+
elif conversion.kind == "date":
|
|
163
|
+
expression = (
|
|
164
|
+
f"CASE WHEN regexp_full_match({base}, '[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}') "
|
|
165
|
+
f"THEN try_cast({base} AS DATE) END"
|
|
166
|
+
)
|
|
167
|
+
else:
|
|
168
|
+
# Preserve explicit instants; never guess a zone for naive timestamps.
|
|
169
|
+
pattern = (
|
|
170
|
+
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}"
|
|
171
|
+
r"(\.[0-9]{1,6})?(Z|[+-][0-9]{2}:[0-9]{2})"
|
|
172
|
+
)
|
|
173
|
+
expression = (
|
|
174
|
+
f"CASE WHEN regexp_full_match({base}, '{pattern}') "
|
|
175
|
+
f"THEN try_cast({base} AS TIMESTAMPTZ) END"
|
|
176
|
+
)
|
|
177
|
+
expressions.append(expression + " AS " + quote(conversion.target))
|
|
178
|
+
attempted = f"{base} IS NOT NULL"
|
|
179
|
+
invalid = f"({attempted}) AND ({expression}) IS NULL"
|
|
180
|
+
attempted_n, failures, missing = self.query(
|
|
181
|
+
f"SELECT count(*) FILTER (WHERE {attempted}), count(*) FILTER (WHERE {invalid}), "
|
|
182
|
+
f"count(*) FILTER (WHERE {base} IS NULL) FROM raw"
|
|
183
|
+
)[0]
|
|
184
|
+
ex = self.query(
|
|
185
|
+
f"SELECT __ef_eda_row, {col} FROM raw WHERE {invalid} "
|
|
186
|
+
f"ORDER BY __ef_eda_row LIMIT {self.config.resources.examples}"
|
|
187
|
+
)
|
|
188
|
+
metadata.append(
|
|
189
|
+
{
|
|
190
|
+
"column": conversion.column,
|
|
191
|
+
"target": conversion.target,
|
|
192
|
+
"policy": conversion.model_dump(mode="json"),
|
|
193
|
+
"attempted": attempted_n,
|
|
194
|
+
"invalid": failures,
|
|
195
|
+
"missing": missing,
|
|
196
|
+
"examples": [{"row": r, "original": v} for r, v in ex],
|
|
197
|
+
}
|
|
198
|
+
)
|
|
199
|
+
if failures:
|
|
200
|
+
self.findings.append(
|
|
201
|
+
Finding(
|
|
202
|
+
code="conversion_invalid",
|
|
203
|
+
severity="rule_violation",
|
|
204
|
+
column=conversion.column,
|
|
205
|
+
count=failures,
|
|
206
|
+
denominator=attempted_n,
|
|
207
|
+
reason=(
|
|
208
|
+
"Explicit conversion failed; original values retained; "
|
|
209
|
+
"no rounding or guessed dates."
|
|
210
|
+
),
|
|
211
|
+
examples=tuple({"row": r, "original": v} for r, v in ex),
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
self.query("CREATE TEMP VIEW typed AS SELECT " + ",".join(expressions) + " FROM raw")
|
|
215
|
+
self.schema = {r[0]: r[1] for r in self.query("DESCRIBE typed")}
|
|
216
|
+
for item, conversion in zip(metadata, self.config.conversions, strict=True):
|
|
217
|
+
if conversion.kind in ("date", "timestamp"):
|
|
218
|
+
col = quote(conversion.target)
|
|
219
|
+
low, high = self.query(
|
|
220
|
+
f"SELECT cast(min({col}) AS VARCHAR), cast(max({col}) AS VARCHAR) FROM typed"
|
|
221
|
+
)[0]
|
|
222
|
+
years = self.query(
|
|
223
|
+
f"SELECT extract(year FROM {col}) __ef_year, count(*) __ef_count FROM typed "
|
|
224
|
+
f"WHERE {col} IS NOT NULL GROUP BY 1 ORDER BY 1"
|
|
225
|
+
)
|
|
226
|
+
item["observed_dates"] = {
|
|
227
|
+
"minimum": value(low),
|
|
228
|
+
"maximum": value(high),
|
|
229
|
+
"years": [{"year": int(y), "rows": n} for y, n in years],
|
|
230
|
+
"timezone": "UTC for explicit instants; dates unchanged",
|
|
231
|
+
"cohort_inferred": False,
|
|
232
|
+
}
|
|
233
|
+
return tuple(metadata)
|
|
234
|
+
|
|
235
|
+
def duplicates(self, columns: tuple[str, ...], rule: KeyRule | None = None) -> DuplicateProfile:
|
|
236
|
+
if not set(columns) <= self.raw_schema.keys():
|
|
237
|
+
raise ValueError("duplicate rule references unknown source column")
|
|
238
|
+
names = ",".join(quote(c) for c in columns)
|
|
239
|
+
predicates = []
|
|
240
|
+
for name in columns:
|
|
241
|
+
col, dtype = quote(name), self.raw_schema[name]
|
|
242
|
+
predicates.append(f"{col} IS NULL")
|
|
243
|
+
if dtype in ("DOUBLE", "FLOAT"):
|
|
244
|
+
predicates.append(f"isnan({col})")
|
|
245
|
+
if rule is not None and rule.empty_is_missing and dtype == "VARCHAR":
|
|
246
|
+
predicates.append(f"{col} = ''")
|
|
247
|
+
missing = " OR ".join(predicates)
|
|
248
|
+
incomplete = self.query(f"SELECT count(*) FROM raw WHERE {missing}")[0][0] if rule else 0
|
|
249
|
+
where = f"WHERE NOT ({missing})" if rule and rule.incomplete != "include" else ""
|
|
250
|
+
denominator = self.query(f"SELECT count(*) FROM raw {where}")[0][0]
|
|
251
|
+
repeated = (
|
|
252
|
+
f"SELECT {names}, count(*) __ef_count FROM raw {where} "
|
|
253
|
+
f"GROUP BY {names} HAVING count(*) > 1"
|
|
254
|
+
)
|
|
255
|
+
groups, involved, excess = self.query(
|
|
256
|
+
f"SELECT count(*), coalesce(sum(__ef_count),0), "
|
|
257
|
+
f"coalesce(sum(__ef_count-1),0) FROM ({repeated})"
|
|
258
|
+
)[0]
|
|
259
|
+
examples = self.query(
|
|
260
|
+
f"SELECT {names}, __ef_count FROM ({repeated}) ORDER BY __ef_count DESC, {names} "
|
|
261
|
+
f"LIMIT {self.config.resources.examples}"
|
|
262
|
+
)
|
|
263
|
+
# Group examples include bounded row references without loading groups in Python.
|
|
264
|
+
references = self.query(
|
|
265
|
+
f"SELECT __ef_eda_row FROM (SELECT *, count(*) OVER "
|
|
266
|
+
f"(PARTITION BY {names}) __ef_n FROM raw {where}) "
|
|
267
|
+
f"WHERE __ef_n > 1 ORDER BY __ef_eda_row LIMIT {self.config.resources.examples}"
|
|
268
|
+
)
|
|
269
|
+
if rule is not None and incomplete and rule.incomplete == "reject":
|
|
270
|
+
self.findings.append(
|
|
271
|
+
Finding(
|
|
272
|
+
code="incomplete_key",
|
|
273
|
+
severity="rule_violation",
|
|
274
|
+
count=incomplete,
|
|
275
|
+
denominator=denominator + incomplete,
|
|
276
|
+
reason="Declared key policy rejects incomplete keys; source retained.",
|
|
277
|
+
examples=self.examples(missing, columns),
|
|
278
|
+
)
|
|
279
|
+
)
|
|
280
|
+
if groups:
|
|
281
|
+
self.findings.append(
|
|
282
|
+
Finding(
|
|
283
|
+
code="key_duplicates" if rule else "row_duplicates",
|
|
284
|
+
severity="rule_violation" if rule and rule.require_unique else "observation",
|
|
285
|
+
count=int(excess),
|
|
286
|
+
denominator=denominator,
|
|
287
|
+
reason="Repetitions observed; may be legitimate history. No row was removed.",
|
|
288
|
+
examples=tuple({"row": r[0]} for r in references),
|
|
289
|
+
)
|
|
290
|
+
)
|
|
291
|
+
return DuplicateProfile(
|
|
292
|
+
columns=columns,
|
|
293
|
+
denominator=denominator,
|
|
294
|
+
incomplete_rows=incomplete,
|
|
295
|
+
policy=rule.incomplete if rule else "all_rows_including_null_and_nan",
|
|
296
|
+
repeated_groups=int(groups),
|
|
297
|
+
involved_rows=int(involved),
|
|
298
|
+
excess_rows=int(excess),
|
|
299
|
+
excess_rate=float(excess / denominator) if denominator else None,
|
|
300
|
+
examples=tuple(
|
|
301
|
+
{
|
|
302
|
+
"values": {c: value(v) for c, v in zip(columns, row[:-1], strict=True)},
|
|
303
|
+
"rows": row[-1],
|
|
304
|
+
}
|
|
305
|
+
for row in examples
|
|
306
|
+
),
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
def numeric(self, rule: NumericRule) -> dict[str, object]:
|
|
310
|
+
if rule.column not in self.schema or not set(rule.groups) <= self.schema.keys():
|
|
311
|
+
raise ValueError("numeric rule references unknown column")
|
|
312
|
+
dtype, col = self.schema[rule.column], quote(rule.column)
|
|
313
|
+
if dtype not in (
|
|
314
|
+
"DOUBLE",
|
|
315
|
+
"FLOAT",
|
|
316
|
+
"INTEGER",
|
|
317
|
+
"BIGINT",
|
|
318
|
+
"SMALLINT",
|
|
319
|
+
"TINYINT",
|
|
320
|
+
"HUGEINT",
|
|
321
|
+
"UBIGINT",
|
|
322
|
+
"UINTEGER",
|
|
323
|
+
"UTINYINT",
|
|
324
|
+
"USMALLINT",
|
|
325
|
+
) and not dtype.startswith("DECIMAL"):
|
|
326
|
+
raise ValueError("numeric column must be numeric or explicitly converted")
|
|
327
|
+
if rule.monetary and not dtype.startswith("DECIMAL"):
|
|
328
|
+
raise ValueError("monetary diagnostics require exact Decimal, never float")
|
|
329
|
+
if rule.monetary and (
|
|
330
|
+
rule.currency_column is None or self.schema[rule.currency_column] != "VARCHAR"
|
|
331
|
+
):
|
|
332
|
+
raise ValueError("currency identifiers must be explicit text, not numeric groups")
|
|
333
|
+
if any(
|
|
334
|
+
self.schema[g]
|
|
335
|
+
not in (
|
|
336
|
+
"VARCHAR",
|
|
337
|
+
"BOOLEAN",
|
|
338
|
+
"INTEGER",
|
|
339
|
+
"BIGINT",
|
|
340
|
+
"FLOAT",
|
|
341
|
+
"DOUBLE",
|
|
342
|
+
"UTINYINT",
|
|
343
|
+
"USMALLINT",
|
|
344
|
+
"SMALLINT",
|
|
345
|
+
"TINYINT",
|
|
346
|
+
"UINTEGER",
|
|
347
|
+
"UBIGINT",
|
|
348
|
+
)
|
|
349
|
+
and not self.schema[g].startswith("DECIMAL")
|
|
350
|
+
for g in rule.groups
|
|
351
|
+
):
|
|
352
|
+
raise CapabilityError("MVP comparable groups admit text/bool/integer/Decimal/float")
|
|
353
|
+
names = ",".join(quote(g) for g in rule.groups)
|
|
354
|
+
group_count = (
|
|
355
|
+
self.query(f"SELECT count(*) FROM (SELECT {names} FROM typed GROUP BY {names})")[0][0]
|
|
356
|
+
if names
|
|
357
|
+
else 1
|
|
358
|
+
)
|
|
359
|
+
if group_count > self.config.resources.max_groups:
|
|
360
|
+
self.findings.append(
|
|
361
|
+
Finding(
|
|
362
|
+
code="group_limit",
|
|
363
|
+
severity="not_evaluable",
|
|
364
|
+
column=rule.column,
|
|
365
|
+
reason=(
|
|
366
|
+
"Comparable group count exceeds configured budget; "
|
|
367
|
+
"no groups silently discarded."
|
|
368
|
+
),
|
|
369
|
+
)
|
|
370
|
+
)
|
|
371
|
+
return {
|
|
372
|
+
"rule": rule.model_dump(mode="json"),
|
|
373
|
+
"state": "not_evaluable",
|
|
374
|
+
"reason": "group_limit",
|
|
375
|
+
"groups_total": group_count,
|
|
376
|
+
}
|
|
377
|
+
groups = (
|
|
378
|
+
self.query(f"SELECT {names} FROM typed GROUP BY {names} ORDER BY {names}")
|
|
379
|
+
if names
|
|
380
|
+
else [()]
|
|
381
|
+
)
|
|
382
|
+
output = []
|
|
383
|
+
for group in groups:
|
|
384
|
+
where = (
|
|
385
|
+
" AND ".join(f"{quote(c)} IS NOT DISTINCT FROM ?" for c in rule.groups) or "TRUE"
|
|
386
|
+
)
|
|
387
|
+
float_type = dtype in ("DOUBLE", "FLOAT")
|
|
388
|
+
finite = f"{col} IS NOT NULL" + (f" AND isfinite({col})" if float_type else "")
|
|
389
|
+
valid_where = f"({where}) AND ({finite})"
|
|
390
|
+
total, valid, missing, nonfinite = self.query(
|
|
391
|
+
f"SELECT count(*), count(*) FILTER (WHERE {finite}), "
|
|
392
|
+
f"count(*) FILTER (WHERE {col} IS NULL), "
|
|
393
|
+
f"count(*) FILTER (WHERE {col} IS NOT NULL AND NOT ({finite})) "
|
|
394
|
+
f"FROM typed WHERE {where}",
|
|
395
|
+
list(group),
|
|
396
|
+
)[0]
|
|
397
|
+
record: dict[str, object] = {
|
|
398
|
+
"group": {c: value(v) for c, v in zip(rule.groups, group, strict=True)},
|
|
399
|
+
"total": total,
|
|
400
|
+
"valid": valid,
|
|
401
|
+
"missing": missing,
|
|
402
|
+
"nonfinite": nonfinite,
|
|
403
|
+
"state": "completed" if valid else "not_evaluable",
|
|
404
|
+
"reason": None if valid else "no_finite_values",
|
|
405
|
+
}
|
|
406
|
+
incomparable = (
|
|
407
|
+
rule.monetary and group[rule.groups.index(rule.currency_column)] in (None, "")
|
|
408
|
+
) or any(isinstance(v, float) and not math.isfinite(v) for v in group)
|
|
409
|
+
if incomparable:
|
|
410
|
+
record.update(
|
|
411
|
+
state="not_evaluable",
|
|
412
|
+
reason="missing_currency" if rule.monetary else "nonfinite_group",
|
|
413
|
+
outliers={
|
|
414
|
+
"method": rule.outliers,
|
|
415
|
+
"state": "not_evaluable",
|
|
416
|
+
"count": None,
|
|
417
|
+
"reason": "missing_currency" if rule.monetary else "nonfinite_group",
|
|
418
|
+
},
|
|
419
|
+
)
|
|
420
|
+
self.findings.append(
|
|
421
|
+
Finding(
|
|
422
|
+
code="incomparable_group",
|
|
423
|
+
severity="not_evaluable",
|
|
424
|
+
column=rule.column,
|
|
425
|
+
denominator=valid,
|
|
426
|
+
reason=(
|
|
427
|
+
"Grouping has missing currency or nonfinite values; "
|
|
428
|
+
"no pooled numeric amounts."
|
|
429
|
+
),
|
|
430
|
+
)
|
|
431
|
+
)
|
|
432
|
+
output.append(record)
|
|
433
|
+
continue
|
|
434
|
+
if not valid:
|
|
435
|
+
output.append(record)
|
|
436
|
+
continue
|
|
437
|
+
percentiles = sorted(set((0.25, 0.5, 0.75, *self.config.percentiles)))
|
|
438
|
+
qs = ",".join(f"quantile_disc({col}, {p})" for p in percentiles)
|
|
439
|
+
sums = self.query(
|
|
440
|
+
f"SELECT min({col}), max({col}), sum({col}), avg({col}), "
|
|
441
|
+
f"stddev_{'pop' if self.config.stddev == 'population' else 'samp'}({col}), {qs} "
|
|
442
|
+
f"FROM typed WHERE {valid_where}",
|
|
443
|
+
list(group),
|
|
444
|
+
)[0]
|
|
445
|
+
qmap = dict(zip(percentiles, sums[5:], strict=True))
|
|
446
|
+
record.update(
|
|
447
|
+
minimum=value(sums[0]),
|
|
448
|
+
maximum=value(sums[1]),
|
|
449
|
+
sum=value(sums[2]),
|
|
450
|
+
mean_approximate=value(sums[3]),
|
|
451
|
+
stddev_approximate=value(sums[4]),
|
|
452
|
+
stddev_definition=self.config.stddev,
|
|
453
|
+
quantiles={str(p): value(qmap[p]) for p in percentiles},
|
|
454
|
+
median=value(qmap[0.5]),
|
|
455
|
+
quantile_method=(
|
|
456
|
+
"DuckDB quantile_disc, Hyndman-Fan type 1; observed value, no interpolation"
|
|
457
|
+
),
|
|
458
|
+
precision=(
|
|
459
|
+
"Exact sum/min/max/discrete quantiles for integer/Decimal; "
|
|
460
|
+
"mean/stddev float64 approximate; float aggregates approximate"
|
|
461
|
+
),
|
|
462
|
+
excluded_from_statistics=(
|
|
463
|
+
"NULL and nonfinite; failed conversions remain separate in conversion receipt"
|
|
464
|
+
),
|
|
465
|
+
)
|
|
466
|
+
if isinstance(sums[2], (int, Decimal)):
|
|
467
|
+
mean = Fraction(sums[2]) / valid
|
|
468
|
+
record["mean_exact_rational"] = {
|
|
469
|
+
"numerator": str(mean.numerator),
|
|
470
|
+
"denominator": str(mean.denominator),
|
|
471
|
+
}
|
|
472
|
+
outlier: dict[str, object] = {
|
|
473
|
+
"method": rule.outliers,
|
|
474
|
+
"multiplier": rule.multiplier,
|
|
475
|
+
"min_support": rule.min_support,
|
|
476
|
+
"state": "not_evaluable",
|
|
477
|
+
"count": None,
|
|
478
|
+
"reason": "disabled" if rule.outliers == "none" else "insufficient_support",
|
|
479
|
+
}
|
|
480
|
+
if valid >= rule.min_support and rule.outliers != "none":
|
|
481
|
+
median = qmap[0.5]
|
|
482
|
+
with localcontext() as ctx:
|
|
483
|
+
ctx.prec = 80
|
|
484
|
+
if rule.outliers == "iqr":
|
|
485
|
+
dispersion = Decimal(str(qmap[0.75])) - Decimal(str(qmap[0.25]))
|
|
486
|
+
lower = (
|
|
487
|
+
Decimal(str(qmap[0.25])) - Decimal(str(rule.multiplier)) * dispersion
|
|
488
|
+
)
|
|
489
|
+
upper = (
|
|
490
|
+
Decimal(str(qmap[0.75])) + Decimal(str(rule.multiplier)) * dispersion
|
|
491
|
+
)
|
|
492
|
+
else:
|
|
493
|
+
mad = self.query(
|
|
494
|
+
f"SELECT quantile_disc(abs({col} - ?),0.5) "
|
|
495
|
+
f"FROM typed WHERE {valid_where}",
|
|
496
|
+
[median, *group],
|
|
497
|
+
)[0][0]
|
|
498
|
+
dispersion = Decimal(str(mad))
|
|
499
|
+
lower = Decimal(str(median)) - Decimal(str(rule.multiplier)) * dispersion
|
|
500
|
+
upper = Decimal(str(median)) + Decimal(str(rule.multiplier)) * dispersion
|
|
501
|
+
if dispersion == 0:
|
|
502
|
+
outlier["reason"] = "zero_dispersion"
|
|
503
|
+
else:
|
|
504
|
+
predicate = f"{valid_where} AND ({col} < ? OR {col} > ?)"
|
|
505
|
+
# Reduce only transient bounds by exact trailing-zero removal,
|
|
506
|
+
# within the high-precision local context, never original money.
|
|
507
|
+
lower = lower.normalize()
|
|
508
|
+
upper = upper.normalize()
|
|
509
|
+
if int(lower.as_tuple().exponent) > 0:
|
|
510
|
+
lower = Decimal(format(lower, "f"))
|
|
511
|
+
if int(upper.as_tuple().exponent) > 0:
|
|
512
|
+
upper = Decimal(format(upper, "f"))
|
|
513
|
+
params = [*group, lower, upper]
|
|
514
|
+
try:
|
|
515
|
+
count = self.query(
|
|
516
|
+
f"SELECT count(*) FROM typed WHERE {predicate}", params
|
|
517
|
+
)[0][0]
|
|
518
|
+
ex = self.examples(predicate, (rule.column, *rule.groups), params)
|
|
519
|
+
except CapabilityError:
|
|
520
|
+
outlier["reason"] = "unrepresentable_decimal_bounds"
|
|
521
|
+
record["outliers"] = outlier
|
|
522
|
+
self.findings.append(
|
|
523
|
+
Finding(
|
|
524
|
+
code="outliers_unavailable",
|
|
525
|
+
severity="not_evaluable",
|
|
526
|
+
column=rule.column,
|
|
527
|
+
denominator=valid,
|
|
528
|
+
reason=(
|
|
529
|
+
"Exact outlier bounds cannot be admitted by DuckDB; "
|
|
530
|
+
"no float fallback."
|
|
531
|
+
),
|
|
532
|
+
)
|
|
533
|
+
)
|
|
534
|
+
output.append(record)
|
|
535
|
+
continue
|
|
536
|
+
outlier.update(
|
|
537
|
+
state="completed",
|
|
538
|
+
reason=None,
|
|
539
|
+
count=count,
|
|
540
|
+
lower=str(lower),
|
|
541
|
+
upper=str(upper),
|
|
542
|
+
dispersion=str(dispersion),
|
|
543
|
+
examples=list(ex),
|
|
544
|
+
denominator=valid,
|
|
545
|
+
interpretation=(
|
|
546
|
+
"Case for investigation; not confirmed error; unscaled MAD or IQR"
|
|
547
|
+
),
|
|
548
|
+
)
|
|
549
|
+
if count:
|
|
550
|
+
self.findings.append(
|
|
551
|
+
Finding(
|
|
552
|
+
code="outlier_candidate",
|
|
553
|
+
severity="observation",
|
|
554
|
+
column=rule.column,
|
|
555
|
+
count=count,
|
|
556
|
+
denominator=valid,
|
|
557
|
+
reason=(
|
|
558
|
+
"Outside configured bounds in a comparable group; "
|
|
559
|
+
"legitimate extremes remain intact."
|
|
560
|
+
),
|
|
561
|
+
examples=ex,
|
|
562
|
+
)
|
|
563
|
+
)
|
|
564
|
+
record["outliers"] = outlier
|
|
565
|
+
if outlier["state"] == "not_evaluable" and rule.outliers != "none":
|
|
566
|
+
self.findings.append(
|
|
567
|
+
Finding(
|
|
568
|
+
code="outliers_unavailable",
|
|
569
|
+
severity="not_evaluable",
|
|
570
|
+
column=rule.column,
|
|
571
|
+
denominator=valid,
|
|
572
|
+
reason=str(outlier["reason"]),
|
|
573
|
+
)
|
|
574
|
+
)
|
|
575
|
+
if not incomparable:
|
|
576
|
+
# Histogram is full-population; bounds/bins use declared float visualization only.
|
|
577
|
+
hist = self.query(
|
|
578
|
+
f"SELECT b, count(*) FROM (SELECT CASE WHEN ? = ? THEN 0 ELSE "
|
|
579
|
+
f"least({self.config.resources.histogram_bins - 1}, greatest(0, "
|
|
580
|
+
f"floor((cast({col} AS DOUBLE)-?) / (? - ?) * "
|
|
581
|
+
f"{self.config.resources.histogram_bins}))) END b FROM typed "
|
|
582
|
+
f"WHERE {valid_where}) GROUP BY b ORDER BY b",
|
|
583
|
+
[
|
|
584
|
+
float(sums[0]),
|
|
585
|
+
float(sums[1]),
|
|
586
|
+
float(sums[0]),
|
|
587
|
+
float(sums[1]),
|
|
588
|
+
float(sums[0]),
|
|
589
|
+
*group,
|
|
590
|
+
],
|
|
591
|
+
)
|
|
592
|
+
record["histogram"] = [{"bin": int(b), "count": n} for b, n in hist]
|
|
593
|
+
record["histogram_bounds_approximate"] = [float(sums[0]), float(sums[1])]
|
|
594
|
+
sample = self.query(
|
|
595
|
+
f"SELECT __ef_eda_row, {col} FROM typed WHERE {valid_where} "
|
|
596
|
+
f"ORDER BY __ef_eda_row LIMIT {self.config.resources.chart_rows}",
|
|
597
|
+
list(group),
|
|
598
|
+
)
|
|
599
|
+
record["chart_sample"] = [{"row": r, "value": value(v)} for r, v in sample]
|
|
600
|
+
record["chart_scope"] = (
|
|
601
|
+
"Deterministic first finite rows in source order; visualization only, "
|
|
602
|
+
"not representative; all statistics calculated on full group"
|
|
603
|
+
)
|
|
604
|
+
output.append(record)
|
|
605
|
+
return {
|
|
606
|
+
"rule": rule.model_dump(mode="json"),
|
|
607
|
+
"groups_total": group_count,
|
|
608
|
+
"groups": output,
|
|
609
|
+
"state": "not_evaluable"
|
|
610
|
+
if any(r["state"] != "completed" for r in output)
|
|
611
|
+
else "completed",
|
|
612
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Explicit first EDA public surface; update only with reviewed contract changes."""
|
|
2
|
+
|
|
3
|
+
INVENTORY = {
|
|
4
|
+
"status": "experimental",
|
|
5
|
+
"format_version": "1.0",
|
|
6
|
+
"public_names": (
|
|
7
|
+
"ColumnProfile",
|
|
8
|
+
"Conversion",
|
|
9
|
+
"DuplicateProfile",
|
|
10
|
+
"EdaConfig",
|
|
11
|
+
"EdaManifest",
|
|
12
|
+
"EdaResult",
|
|
13
|
+
"Finding",
|
|
14
|
+
"KeyRule",
|
|
15
|
+
"NumericRule",
|
|
16
|
+
"Resources",
|
|
17
|
+
"ResultFile",
|
|
18
|
+
"Source",
|
|
19
|
+
"analyze",
|
|
20
|
+
"analyze_artifact",
|
|
21
|
+
"analyze_migration",
|
|
22
|
+
"read_result",
|
|
23
|
+
),
|
|
24
|
+
"signatures": {
|
|
25
|
+
"analyze": "(source: 'Source', *, config: 'EdaConfig | None' = None, "
|
|
26
|
+
"output: 'Path', cancellation: 'Callable[[], bool] | None' = "
|
|
27
|
+
"None) -> 'EdaResult'",
|
|
28
|
+
"analyze_artifact": "(artifact: 'TableArtifact', *, config: 'EdaConfig | "
|
|
29
|
+
"None' = None, output: 'Path') -> 'EdaResult'",
|
|
30
|
+
"analyze_migration": "(path: 'Path', *, side: \"Literal['source', "
|
|
31
|
+
"'target']\" = 'source', config: 'EdaConfig | "
|
|
32
|
+
"None' = None, output: 'Path') -> 'EdaResult'",
|
|
33
|
+
"read_result": "(path: 'Path') -> 'EdaResult'",
|
|
34
|
+
},
|
|
35
|
+
"model_shapes": {
|
|
36
|
+
"ColumnProfile": "fef8bbba53cfe502ec13bdbdb6c20e2635a1c9865b92655cebe367ace0ce81e9",
|
|
37
|
+
"Conversion": "f6f432c6c02a2f3f770d6fc602cf6dc6202302b46bdf670540cc2f2dadc00f95",
|
|
38
|
+
"DuplicateProfile": "2a1f6d9efd1618d82de32b4c4d5948f3a0d89b0452924da3c495c69caa9eeaa6",
|
|
39
|
+
"EdaConfig": "65ea2de1dd1d43bd0f1bb8942dd5b28ff7a95f346c24edc01418ec5a04083fe1",
|
|
40
|
+
"EdaManifest": "f9d3b6ccb63249f7bd796f8154135a04c844578231666a988e400c857d0b3e2d",
|
|
41
|
+
"EdaResult": "5bce9ab1457ab99b5ed818ecaac7695caef205c12eff29dd0fe6507e8c8a0cad",
|
|
42
|
+
"Finding": "331db5cf15d9e26ef8b94e2fd25e625e2e804165f6c584d78a6a3c83ea41faa2",
|
|
43
|
+
"KeyRule": "4b0f8f2f6baf3eca22ab5c0da607d90f46c61bec008b5d8a760b61372a18828e",
|
|
44
|
+
"NumericRule": "16951b2437617d4d52437d3124d7b394f546ba49eac98f9c049ab07fefe21938",
|
|
45
|
+
"Resources": "82173ef020b2937c4c7aaf1cea037effbc836dfecf25e5929c29cd3702898e9a",
|
|
46
|
+
"ResultFile": "eb3f6aa3e82b85493541404add0212327cf0b6223f2c858cd615d83df1e1b7e8",
|
|
47
|
+
"Source": "182d1cc45e149e0251acdae721dd78b4f73721d69cd52f630140c805b96265fb",
|
|
48
|
+
},
|
|
49
|
+
}
|