quaxdim 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. quaxdim-0.9.0/CHANGELOG.md +190 -0
  2. quaxdim-0.9.0/CONTRIBUTING.md +26 -0
  3. quaxdim-0.9.0/LICENSE +21 -0
  4. quaxdim-0.9.0/MANIFEST.in +16 -0
  5. quaxdim-0.9.0/PKG-INFO +470 -0
  6. quaxdim-0.9.0/README.md +403 -0
  7. quaxdim-0.9.0/SECURITY.md +13 -0
  8. quaxdim-0.9.0/contracts/quality_contract.json +430 -0
  9. quaxdim-0.9.0/contracts/quality_contract.yaml +315 -0
  10. quaxdim-0.9.0/docs/README.es.md +43 -0
  11. quaxdim-0.9.0/docs/architecture.es.md +254 -0
  12. quaxdim-0.9.0/docs/dimensions.es.md +71 -0
  13. quaxdim-0.9.0/docs/execution.es.md +94 -0
  14. quaxdim-0.9.0/docs/formats_catalog.csv +57 -0
  15. quaxdim-0.9.0/docs/statistical_validity.md +112 -0
  16. quaxdim-0.9.0/examples/advanced_recommendation.py +18 -0
  17. quaxdim-0.9.0/examples/auto_recommend.py +50 -0
  18. quaxdim-0.9.0/examples/contract_employees.yaml +108 -0
  19. quaxdim-0.9.0/examples/demo.py +34 -0
  20. quaxdim-0.9.0/examples/employees.csv +7 -0
  21. quaxdim-0.9.0/examples/execute_contract.py +50 -0
  22. quaxdim-0.9.0/examples/execution_demo_report.json +205 -0
  23. quaxdim-0.9.0/examples/metadata.example.json +1 -0
  24. quaxdim-0.9.0/examples/plan_applicability.py +32 -0
  25. quaxdim-0.9.0/examples/quality.yaml +69 -0
  26. quaxdim-0.9.0/examples/resultado_plan_demo.json +233 -0
  27. quaxdim-0.9.0/examples/spark_execute.py +70 -0
  28. quaxdim-0.9.0/pyproject.toml +107 -0
  29. quaxdim-0.9.0/setup.cfg +4 -0
  30. quaxdim-0.9.0/src/quaxdim/__init__.py +82 -0
  31. quaxdim-0.9.0/src/quaxdim/__main__.py +6 -0
  32. quaxdim-0.9.0/src/quaxdim/_io.py +58 -0
  33. quaxdim-0.9.0/src/quaxdim/_lazy.py +45 -0
  34. quaxdim-0.9.0/src/quaxdim/_types.py +16 -0
  35. quaxdim-0.9.0/src/quaxdim/_version.py +3 -0
  36. quaxdim-0.9.0/src/quaxdim/advanced_discovery.py +1817 -0
  37. quaxdim-0.9.0/src/quaxdim/applicability.py +1116 -0
  38. quaxdim-0.9.0/src/quaxdim/approval.py +211 -0
  39. quaxdim-0.9.0/src/quaxdim/bounds.py +79 -0
  40. quaxdim-0.9.0/src/quaxdim/catalog.py +316 -0
  41. quaxdim-0.9.0/src/quaxdim/cli.py +431 -0
  42. quaxdim-0.9.0/src/quaxdim/contract_validation.py +148 -0
  43. quaxdim-0.9.0/src/quaxdim/core.py +1055 -0
  44. quaxdim-0.9.0/src/quaxdim/csv_dialect.py +303 -0
  45. quaxdim-0.9.0/src/quaxdim/discovery.py +914 -0
  46. quaxdim-0.9.0/src/quaxdim/evaluator_registry.py +191 -0
  47. quaxdim-0.9.0/src/quaxdim/execution.py +938 -0
  48. quaxdim-0.9.0/src/quaxdim/heuristics.py +339 -0
  49. quaxdim-0.9.0/src/quaxdim/missingness.py +68 -0
  50. quaxdim-0.9.0/src/quaxdim/optimized.py +1570 -0
  51. quaxdim-0.9.0/src/quaxdim/pandas_evaluators.py +212 -0
  52. quaxdim-0.9.0/src/quaxdim/profile.py +50 -0
  53. quaxdim-0.9.0/src/quaxdim/py.typed +0 -0
  54. quaxdim-0.9.0/src/quaxdim/runtime.py +52 -0
  55. quaxdim-0.9.0/src/quaxdim/schemas/quality-contract.schema.json +1215 -0
  56. quaxdim-0.9.0/src/quaxdim/spark_backend.py +1116 -0
  57. quaxdim-0.9.0/src/quaxdim/temporal.py +120 -0
  58. quaxdim-0.9.0/src/quaxdim.egg-info/PKG-INFO +470 -0
  59. quaxdim-0.9.0/src/quaxdim.egg-info/SOURCES.txt +82 -0
  60. quaxdim-0.9.0/src/quaxdim.egg-info/dependency_links.txt +1 -0
  61. quaxdim-0.9.0/src/quaxdim.egg-info/entry_points.txt +2 -0
  62. quaxdim-0.9.0/src/quaxdim.egg-info/requires.txt +47 -0
  63. quaxdim-0.9.0/src/quaxdim.egg-info/top_level.txt +1 -0
  64. quaxdim-0.9.0/tests/test_advanced_discovery.py +307 -0
  65. quaxdim-0.9.0/tests/test_applicability.py +439 -0
  66. quaxdim-0.9.0/tests/test_discovery.py +267 -0
  67. quaxdim-0.9.0/tests/test_dispatch.py +32 -0
  68. quaxdim-0.9.0/tests/test_execution.py +483 -0
  69. quaxdim-0.9.0/tests/test_quality.py +286 -0
  70. quaxdim-0.9.0/tests/test_spark_backend.py +228 -0
  71. quaxdim-0.9.0/tests_public/test_public_api.py +72 -0
  72. quaxdim-0.9.0/tests_public/test_without_pandas.py +136 -0
  73. quaxdim-0.9.0/tests_validation/test_accuracy_benchmark.py +47 -0
  74. quaxdim-0.9.0/tests_validation/test_audit_regressions.py +308 -0
  75. quaxdim-0.9.0/tests_validation/test_backend_parity.py +44 -0
  76. quaxdim-0.9.0/tests_validation/test_contract_and_safety.py +87 -0
  77. quaxdim-0.9.0/tests_validation/test_csv_acceptance.py +183 -0
  78. quaxdim-0.9.0/tests_validation/test_discovery_heuristics.py +207 -0
  79. quaxdim-0.9.0/tests_validation/test_optional_engines.py +146 -0
  80. quaxdim-0.9.0/tests_validation/test_pandas_readonly_masks.py +44 -0
  81. quaxdim-0.9.0/tests_validation/test_read_options_parity.py +458 -0
  82. quaxdim-0.9.0/tests_validation/test_v08_native_optional.py +261 -0
  83. quaxdim-0.9.0/tests_validation/test_v08_optimized.py +304 -0
  84. quaxdim-0.9.0/tests_validation/test_v08_sql_compiler.py +71 -0
@@ -0,0 +1,190 @@
1
+ # Changelog
2
+
3
+ ## Unreleased
4
+
5
+ ### Audit fixes — Phase 6 (English messages and infinity policy)
6
+
7
+ - **Behaviour change:** every user-facing string produced by the library
8
+ is now English: recommendation titles, rationales, caveats and
9
+ prerequisites of both advisors, and the dimension catalogue
10
+ (`DimensionSpec.name_es` is now `name`; `dimension_es` in legacy result
11
+ dictionaries is now `dimension_name`). Code that matched Spanish titles
12
+ must be updated.
13
+ - `missing_policy.infinities_are_invalid` is enforced: when true, an
14
+ infinite value in a float field fails every rule on that field on
15
+ DuckDB, Polars and Spark; the pandas planner blocks such rules as
16
+ `UNSUPPORTED` instead of passing `+inf` silently. Default stays false
17
+ (infinities compare as numbers).
18
+
19
+ ### Audit fixes — Phase 5 (documentation and validation)
20
+
21
+ - New `docs/statistical_validity.md`: identification bounds vs confidence
22
+ intervals, per-row uniqueness, thresholds as census proportions, missing
23
+ values as declared policy, quantified discovery evidence, drift as a
24
+ heuristic, no composite index, tested engine equivalence.
25
+ - New `benchmarks/discovery_accuracy.py` with ground truth: discovery
26
+ precision/recall under injected violations and HyperLogLog bias/RMSE
27
+ versus theory, with a smoke test in `tests_validation/`.
28
+ - README capability table aligned with what is tested.
29
+
30
+ ### Deferred (not in this branch)
31
+
32
+ - Splitting `advanced_discovery.py` by source (`sources/csv.py`, ...): a
33
+ pure code move, left for its own commit so that it does not mix with
34
+ semantic changes.
35
+ - Spark execution could not be run in the fixing environment; the Spark
36
+ paths changed only where shared code changed and must be run in CI.
37
+
38
+ ### Audit fixes — Phase 4 (API robustness and engineering)
39
+
40
+ - `QualitySuite.add()` validates before mutating; a rejected check leaves
41
+ the suite unchanged (audit H3). `schema_columns` reports duplicate
42
+ column names instead of hiding them behind set comparison (audit H4).
43
+ - The no-raw-values test excludes the report timestamp, whose
44
+ microseconds could contain the probed value (audit H8).
45
+ - Reports carry `library_version` and a `sources` audit trail (file path,
46
+ size and modification time, or in-memory type) next to
47
+ `contract_sha256`; thresholds were already per rule.
48
+ - Every report, contract and suite file is written atomically (temporary
49
+ file + `os.replace`): a crash mid-write leaves the previous file intact.
50
+ - `quaxdim --version`. Known errors still exit with code 2 and a failing
51
+ native execution with code 1.
52
+ - Optional dependencies get upper bounds (`pandas<3`, `duckdb<3`,
53
+ `polars<3`); Polars 1.x and 2.x and DuckDB 2.0 are tested.
54
+ - Verified: the wheel and the sdist do not ship `assets/` (the restricted
55
+ logo stays out of PyPI).
56
+
57
+ ### Audit fixes — Phase 3 (discovery heuristics)
58
+
59
+ - New `heuristics` module: one set of name semantics and thresholds for
60
+ both advisors (`discover` and `discover_advanced`), exposed as
61
+ `HeuristicThresholds` and `DiscoveryBudget.thresholds`. The divergent
62
+ values are unified: closed-domain share 0.25 (was 0.25 / 0.35), code
63
+ width up to 40 (was 30 / 40).
64
+ - Semantic roles and start/end pairs match name **tokens** (snake_case,
65
+ camelCase, spaces, accents folded) instead of substrings. `country`,
66
+ `account_balance`, `tamaño_empresa` and `candidate_id` no longer get
67
+ wrong roles, and start/end fields pair only when they share the event
68
+ root (audit A3).
69
+ - Key candidates use a denominator without blanks, so a few blanks no
70
+ longer hide a unique key (audit A1).
71
+ - The HLL uniqueness hypothesis tolerates `z · relative_std_error` below a
72
+ ratio of one, following `hll_precision`, instead of a fixed
73
+ `[0.94, 1.06]` window (audit A2).
74
+ - Sample-based evidence is quantified: rule-of-three upper bounds for
75
+ patterns and ISO dates, Good–Turing unseen-category share for domains,
76
+ minimum detectable duplicate share for composite keys, and repeated
77
+ determinants plus the g3 error for functional dependencies, which now
78
+ need at least three repeated determinants (audit M1–M3).
79
+ - ISO date hypotheses need 30 matching values for `high` confidence
80
+ (three used to suffice).
81
+
82
+ ### Audit fixes — Phase 2 (reading and engine parity)
83
+
84
+ - New `asset.read_options` (schema addition, backward compatible):
85
+ `delimiter`, `encoding`, `quote_char`, `header`, `malformed_rows`,
86
+ `max_field_chars`. Discovery records the dialect it used and
87
+ `build_reviewed_contract` copies it into the contract, so evaluation reads
88
+ the same table (audit H6). Previously `discover_advanced(csv_delimiter=)`
89
+ had no counterpart in `evaluate_optimized`.
90
+ - DuckDB reads CSV with an explicit, strict dialect (`auto_detect=false`,
91
+ `strict_mode=true`); its sniffer could read a file as a single column.
92
+ - **Behaviour change:** malformed CSV is an ERROR on every engine: rows with
93
+ too few or too many fields, duplicate or empty header names, oversized
94
+ fields. Before, `csv_stream` counted a short row's missing field as an
95
+ ordinary missing value, and DuckDB/Polars silently renamed duplicate
96
+ headers (`A_1`, `A_duplicated_0`); Polars padded short rows.
97
+ - Engines that cannot decode the declared encoding block the rule
98
+ (`UNSUPPORTED`) instead of misreading text.
99
+ - Discovery's DuckDB path uses the same strict dialect; discovery on
100
+ `csv_stream` still profiles malformed files and warns, because it is
101
+ exploratory, but the resulting contract refuses them at evaluation.
102
+ - New parity matrix `tests_validation/test_read_options_parity.py` (BOM,
103
+ CRLF, quoted line breaks and delimiters, doubled quotes, gzip, tokens,
104
+ blanks, duplicates, NaN/inf/null on Parquet) across csv_stream, DuckDB
105
+ and Polars.
106
+
107
+ ### Audit fixes — Phase 1 (verdict correctness)
108
+
109
+ - **Behaviour change:** uniqueness bounds are no longer row-local. An
110
+ unknown key may collide with a unique key and turn both rows into
111
+ failures, so the worst case is `max(passed - unknown, 0) / eligible`.
112
+ Rules that used to PASS with unknown keys can now be INDETERMINATE
113
+ (audit C1). Each evaluator declares its bound model in the registry.
114
+ - **Behaviour change:** `plan_optimized` / `evaluate_optimized` validate
115
+ the contract against the JSON Schema and reject repeated IDs, like the
116
+ pandas planner. Hand-written contracts without `evaluation` or with
117
+ non-semver versions are now rejected with `ContractError` (audit H1).
118
+ - Uniqueness rules whose `expectation.key_fields` differ from
119
+ `target.fields` are INVALID_CONTRACT on every engine; previously the
120
+ optimized engine evaluated a different key and could pass (audit H5).
121
+ - Metrics referring to undeclared dimensions are INVALID_CONTRACT on the
122
+ optimized engine too.
123
+ - **Behaviour change:** empty strings in text fields are missing by
124
+ default on every engine; declare `missing_policy.empty_string: false`
125
+ to keep `""` as a value. DuckDB and Polars CSV readers no longer turn
126
+ empty fields into NULL behind the policy's back (audit H7).
127
+ - NaN detection follows the real engine type and honours
128
+ `missing_policy.nan_is_missing` (default true), instead of trusting the
129
+ declared `physical_type` (audit H2). The SQL predicate also covers a
130
+ DuckDB 2.0 case where `isnan` returns false for NaN read from Parquet.
131
+ - `QualitySuite` is deprecated. `freshness_days` and `delay_days` now
132
+ require `params.date_format` and, for naive timestamps,
133
+ `params.timezone`; formats are never inferred and UTC is never assumed.
134
+ Values that do not match the format make the check ERROR instead of
135
+ counting as late (audit C2).
136
+ - Typing: aggregate queries fail loudly if no row comes back; Polars 2
137
+ `collect` compatibility. `mypy src` is clean again.
138
+ - New regression suite `tests_validation/test_audit_regressions.py`.
139
+
140
+ - **Breaking (never published):** a single `quaxdim` package. The earlier
141
+ `quality_dimensions` import path and the `qd` command were removed; the
142
+ package had not been released to PyPI, so no migration is needed.
143
+ - pandas and numpy are now truly optional. Previously the README quickstart
144
+ commands `native-plan` / `native-execute` and several public names failed
145
+ on a core install because modules imported pandas at import time. They are
146
+ bound lazily and raise an `ImportError` naming the `pandas` extra.
147
+ - Version is defined once in `quaxdim/_version.py`; `pyproject.toml` reads it.
148
+ - Ship `py.typed`; remove the unused `legacy` extra and the duplicated
149
+ contract JSON schema (the packaged copy is the single source).
150
+ - Example contracts renamed to `quality_contract.{json,yaml}`.
151
+ - CI: add a core-only (no pandas) job, wheel-content checks and a blocking
152
+ lint/type-check job.
153
+ - Code quality: formatted to PEP 8 (79 columns); `ruff` (E, W, F, I, N, UP,
154
+ B, D1) and `mypy` (`disallow_untyped_defs`) are clean and enforced in CI;
155
+ every function is annotated and every public class, method and function has
156
+ a docstring.
157
+ - Fix: closures that captured loop variables, unused locals, `zip` without
158
+ `strict`, and a variable reused with two types in `_polars_run`.
159
+ - Fix: error messages pointed to a removed package name; they now name the
160
+ `quaxdim` extras.
161
+ - Behaviour made explicit: an approved rule without a dimension raises
162
+ `ValueError`; `SparkBackendUnavailable` is now `SparkBackendUnavailableError`.
163
+
164
+ ## 0.8.0rc1 — 2026-11-04
165
+
166
+ - Publish under the **QuaxDim** brand with owner-supplied artwork.
167
+ - Add official `quaxdim` Python namespace and CLI command; retain legacy
168
+ `quality_dimensions` imports and `qd` alias.
169
+ - Add packaging metadata, source/wheel manifest, CI, release workflow with
170
+ PyPI OIDC Trusted Publishing, Windows test kit and bilingual docs.
171
+ - Preserve the experimental v0.8 core: streamed discovery, native optimized
172
+ planner and portable evaluation; no claim of full dimension coverage.
173
+ - No changes intended to the underlying metric semantics.
174
+
175
+ ## 0.8.0.dev0 — pre-publication build
176
+
177
+ - Native execution planning for CSV, DuckDB, Polars and Spark with explicit
178
+ approvals and resource limits; broader discovery recommendations.
179
+ - Optional pandas compatibility layer.
180
+
181
+ ## Earlier experimental builds (0.1–0.7.1)
182
+
183
+ Initial dimension catalog, contracts, applicability planner, evaluators,
184
+ recommendation system and native adapters. See [docs](docs/) for details.
185
+
186
+ ## 0.9.0 — 2026-11-06
187
+
188
+ Fixes every finding of the three external audits of 0.8.0rc1. Several
189
+ changes alter verdicts or outputs of existing contracts; each is marked
190
+ **Behaviour change** below. Discovery output is now English.
@@ -0,0 +1,26 @@
1
+ # Contributing to QuaxDim
2
+
3
+ Issues and pull requests are welcome. Keep bug reports reproducible and anonymize
4
+ all data. For new quality rules, document **what is measured**, eligible denominator,
5
+ null/unknown policy, failure behavior, evidence needs and expected backend semantics.
6
+
7
+ ## Local tests
8
+
9
+ ```bash
10
+ python -m pip install -e '.[dev]'
11
+ python -m ruff format src tests && python -m ruff check src && python -m mypy src
12
+ python -m pytest -q tests tests_validation tests_public
13
+ ```
14
+
15
+ Optional engines are tested separately with `.[duckdb,polars,spark,parquet]`.
16
+ Never claim backend parity from a skipped test. Run Spark tests with a supported Java.
17
+
18
+ ## Pull request checklist
19
+
20
+ - Tests cover passes, failures, missing/indeterminate outcomes and invalid contracts.
21
+ - No implicit casts or destructive mutations.
22
+ - Performance impact and extra scans are documented.
23
+ - New rules are conservative if backends disagree.
24
+ - Public API/docs/changelog are updated when needed.
25
+
26
+ Code is licensed MIT. The duck logo is a separately managed brand asset.
quaxdim-0.9.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,16 @@
1
+ include LICENSE README.md CHANGELOG.md CONTRIBUTING.md SECURITY.md
2
+ recursive-include contracts *.json *.yaml
3
+ recursive-include docs *.md *.csv
4
+ recursive-include examples *.py *.yaml *.json *.csv
5
+ recursive-include tests *.py
6
+ recursive-include tests_validation *.py
7
+ recursive-include tests_public *.py
8
+ recursive-include src/quaxdim/schemas *.json
9
+ include src/quaxdim/py.typed
10
+ prune .github
11
+ prune scripts
12
+ prune benchmarks
13
+ prune build
14
+ prune dist
15
+ prune reportes
16
+ global-exclude __pycache__ *.py[cod] .DS_Store *.log *.xml