sheets-data-validation-kit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sheets_data_validation_kit-0.1.0/LICENSE +21 -0
- sheets_data_validation_kit-0.1.0/PKG-INFO +133 -0
- sheets_data_validation_kit-0.1.0/README.md +108 -0
- sheets_data_validation_kit-0.1.0/pyproject.toml +44 -0
- sheets_data_validation_kit-0.1.0/setup.cfg +4 -0
- sheets_data_validation_kit-0.1.0/src/sheets_data_validation_kit.egg-info/PKG-INFO +133 -0
- sheets_data_validation_kit-0.1.0/src/sheets_data_validation_kit.egg-info/SOURCES.txt +15 -0
- sheets_data_validation_kit-0.1.0/src/sheets_data_validation_kit.egg-info/dependency_links.txt +1 -0
- sheets_data_validation_kit-0.1.0/src/sheets_data_validation_kit.egg-info/requires.txt +4 -0
- sheets_data_validation_kit-0.1.0/src/sheets_data_validation_kit.egg-info/top_level.txt +1 -0
- sheets_data_validation_kit-0.1.0/src/validation_kit/__init__.py +16 -0
- sheets_data_validation_kit-0.1.0/src/validation_kit/contract.py +201 -0
- sheets_data_validation_kit-0.1.0/src/validation_kit/entities.py +113 -0
- sheets_data_validation_kit-0.1.0/src/validation_kit/reconcile.py +97 -0
- sheets_data_validation_kit-0.1.0/tests/test_contract.py +80 -0
- sheets_data_validation_kit-0.1.0/tests/test_entities.py +47 -0
- sheets_data_validation_kit-0.1.0/tests/test_reconcile.py +56 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 SK Saimon
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sheets-data-validation-kit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Validation, reconciliation and entity-resolution primitives for spreadsheet and CSV pipelines
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/devops-gm88/sheets-data-validation-kit
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
18
|
+
Requires-Python: >=3.9
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
23
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# sheets-data-validation-kit
|
|
27
|
+
|
|
28
|
+
[](https://github.com/devops-gm88/sheets-data-validation-kit/actions/workflows/ci.yml)
|
|
29
|
+
[](pyproject.toml)
|
|
30
|
+
[](LICENSE)
|
|
31
|
+
|
|
32
|
+
Validation, reconciliation and entity-resolution primitives for spreadsheet and CSV pipelines.
|
|
33
|
+
|
|
34
|
+
Most data pipelines fail quietly. A row lands, a number looks plausible, and nobody notices it is wrong
|
|
35
|
+
until it has been reported to someone. This kit exists to make that failure mode loud.
|
|
36
|
+
|
|
37
|
+
It is deliberately **standard-library only** — no dependencies, nothing to install, nothing to upgrade.
|
|
38
|
+
Copy the `src/validation_kit` folder into your project, or install the package in one line:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install git+https://github.com/devops-gm88/sheets-data-validation-kit.git
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Not on PyPI yet - the GitHub URL above is the install. From a clone:
|
|
45
|
+
`pip install -e .`.
|
|
46
|
+
|
|
47
|
+
## The three ideas
|
|
48
|
+
|
|
49
|
+
**1. Rows pass or they are quarantined. They are never silently dropped.**
|
|
50
|
+
A row that fails a contract goes to quarantine with the reasons attached. You can always answer
|
|
51
|
+
"where did this record go?" — which is the question that kills trust in a pipeline.
|
|
52
|
+
|
|
53
|
+
**2. Arithmetic is a second, independent check.**
|
|
54
|
+
Field-level validation confirms each value looks right. Reconciliation confirms the values are
|
|
55
|
+
*consistent with each other*. An extracted balance sheet must balance; a cash flow statement must tie to
|
|
56
|
+
the change in cash. If the arithmetic does not close, the extraction failed regardless of how confident
|
|
57
|
+
each individual field was. This catches the failure nobody sees: clean extraction, quietly wrong numbers.
|
|
58
|
+
|
|
59
|
+
**3. Exact match or human review. Never a fuzzy merge.**
|
|
60
|
+
Duplicates are detected on a canonical key. Suspected duplicates are queued for a person, because a
|
|
61
|
+
silent merge corrupts counts invisibly and is very hard to unpick later.
|
|
62
|
+
|
|
63
|
+
## Quick start
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from validation_kit import RowContract, Field, Min, Range, Pattern, Reconcile, Assertion
|
|
67
|
+
from validation_kit.entities import resolve
|
|
68
|
+
|
|
69
|
+
contract = RowContract([
|
|
70
|
+
Field("account_id", required=True),
|
|
71
|
+
Field("amount", rules=[Min(0)], coerce=float,
|
|
72
|
+
on_fail="quarantine"),
|
|
73
|
+
Field("period", rules=[Pattern(r"^\d{4}-Q[1-4]$")]),
|
|
74
|
+
])
|
|
75
|
+
|
|
76
|
+
result = contract.validate_rows(rows)
|
|
77
|
+
print(result.summary())
|
|
78
|
+
for row, reasons in result.quarantined:
|
|
79
|
+
print(row.get("account_id"), reasons)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
# Arithmetic self-consistency: the check that does not depend on the model
|
|
84
|
+
suite = Reconcile([
|
|
85
|
+
Assertion("balance_sheet_balances",
|
|
86
|
+
lambda r: r["total_assets"], lambda r: r["total_liabilities"] + r["total_equity"],
|
|
87
|
+
tolerance=0.01),
|
|
88
|
+
])
|
|
89
|
+
|
|
90
|
+
for outcome in suite.check(parsed_statement):
|
|
91
|
+
print(outcome) # PASS / FAIL with both sides of the equation printed
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
# Entity resolution: exact key, review queue, no auto-merge
|
|
96
|
+
outcome = resolve(records, key="child_id")
|
|
97
|
+
print(outcome.unique_count, len(outcome.duplicate_candidates))
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## What this is not
|
|
101
|
+
|
|
102
|
+
- It is not a data-quality *scoring* tool. There is no 0-100 number and no dashboard.
|
|
103
|
+
- It does not fix your source data. It tells you which records are untrustworthy and why.
|
|
104
|
+
- It will not merge duplicates for you. That is the point.
|
|
105
|
+
|
|
106
|
+
## Why it exists
|
|
107
|
+
|
|
108
|
+
Built from repeatedly solving the same problem on real reporting systems: a monthly pack that was
|
|
109
|
+
*almost* right, and no way to prove which numbers were which. These are the primitives that were
|
|
110
|
+
written more than once, extracted so they are not written again.
|
|
111
|
+
|
|
112
|
+
## Tests
|
|
113
|
+
|
|
114
|
+
Install first, then run:
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
pip install -e .
|
|
118
|
+
python3 -m unittest discover -s tests -v
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Or without installing anything:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Both give **24 tests, OK**. Running `unittest` from a fresh clone without one of
|
|
128
|
+
those two steps fails to import the package — that is a packaging path, not a
|
|
129
|
+
test failure.
|
|
130
|
+
|
|
131
|
+
## License
|
|
132
|
+
|
|
133
|
+
MIT
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# sheets-data-validation-kit
|
|
2
|
+
|
|
3
|
+
[](https://github.com/devops-gm88/sheets-data-validation-kit/actions/workflows/ci.yml)
|
|
4
|
+
[](pyproject.toml)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Validation, reconciliation and entity-resolution primitives for spreadsheet and CSV pipelines.
|
|
8
|
+
|
|
9
|
+
Most data pipelines fail quietly. A row lands, a number looks plausible, and nobody notices it is wrong
|
|
10
|
+
until it has been reported to someone. This kit exists to make that failure mode loud.
|
|
11
|
+
|
|
12
|
+
It is deliberately **standard-library only** — no dependencies, nothing to install, nothing to upgrade.
|
|
13
|
+
Copy the `src/validation_kit` folder into your project, or install the package in one line:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install git+https://github.com/devops-gm88/sheets-data-validation-kit.git
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Not on PyPI yet - the GitHub URL above is the install. From a clone:
|
|
20
|
+
`pip install -e .`.
|
|
21
|
+
|
|
22
|
+
## The three ideas
|
|
23
|
+
|
|
24
|
+
**1. Rows pass or they are quarantined. They are never silently dropped.**
|
|
25
|
+
A row that fails a contract goes to quarantine with the reasons attached. You can always answer
|
|
26
|
+
"where did this record go?" — which is the question that kills trust in a pipeline.
|
|
27
|
+
|
|
28
|
+
**2. Arithmetic is a second, independent check.**
|
|
29
|
+
Field-level validation confirms each value looks right. Reconciliation confirms the values are
|
|
30
|
+
*consistent with each other*. An extracted balance sheet must balance; a cash flow statement must tie to
|
|
31
|
+
the change in cash. If the arithmetic does not close, the extraction failed regardless of how confident
|
|
32
|
+
each individual field was. This catches the failure nobody sees: clean extraction, quietly wrong numbers.
|
|
33
|
+
|
|
34
|
+
**3. Exact match or human review. Never a fuzzy merge.**
|
|
35
|
+
Duplicates are detected on a canonical key. Suspected duplicates are queued for a person, because a
|
|
36
|
+
silent merge corrupts counts invisibly and is very hard to unpick later.
|
|
37
|
+
|
|
38
|
+
## Quick start
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from validation_kit import RowContract, Field, Min, Range, Pattern, Reconcile, Assertion
|
|
42
|
+
from validation_kit.entities import resolve
|
|
43
|
+
|
|
44
|
+
contract = RowContract([
|
|
45
|
+
Field("account_id", required=True),
|
|
46
|
+
Field("amount", rules=[Min(0)], coerce=float,
|
|
47
|
+
on_fail="quarantine"),
|
|
48
|
+
Field("period", rules=[Pattern(r"^\d{4}-Q[1-4]$")]),
|
|
49
|
+
])
|
|
50
|
+
|
|
51
|
+
result = contract.validate_rows(rows)
|
|
52
|
+
print(result.summary())
|
|
53
|
+
for row, reasons in result.quarantined:
|
|
54
|
+
print(row.get("account_id"), reasons)
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
# Arithmetic self-consistency: the check that does not depend on the model
|
|
59
|
+
suite = Reconcile([
|
|
60
|
+
Assertion("balance_sheet_balances",
|
|
61
|
+
lambda r: r["total_assets"], lambda r: r["total_liabilities"] + r["total_equity"],
|
|
62
|
+
tolerance=0.01),
|
|
63
|
+
])
|
|
64
|
+
|
|
65
|
+
for outcome in suite.check(parsed_statement):
|
|
66
|
+
print(outcome) # PASS / FAIL with both sides of the equation printed
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
# Entity resolution: exact key, review queue, no auto-merge
|
|
71
|
+
outcome = resolve(records, key="child_id")
|
|
72
|
+
print(outcome.unique_count, len(outcome.duplicate_candidates))
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## What this is not
|
|
76
|
+
|
|
77
|
+
- It is not a data-quality *scoring* tool. There is no 0-100 number and no dashboard.
|
|
78
|
+
- It does not fix your source data. It tells you which records are untrustworthy and why.
|
|
79
|
+
- It will not merge duplicates for you. That is the point.
|
|
80
|
+
|
|
81
|
+
## Why it exists
|
|
82
|
+
|
|
83
|
+
Built from repeatedly solving the same problem on real reporting systems: a monthly pack that was
|
|
84
|
+
*almost* right, and no way to prove which numbers were which. These are the primitives that were
|
|
85
|
+
written more than once, extracted so they are not written again.
|
|
86
|
+
|
|
87
|
+
## Tests
|
|
88
|
+
|
|
89
|
+
Install first, then run:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
pip install -e .
|
|
93
|
+
python3 -m unittest discover -s tests -v
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Or without installing anything:
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Both give **24 tests, OK**. Running `unittest` from a fresh clone without one of
|
|
103
|
+
those two steps fails to import the package — that is a packaging path, not a
|
|
104
|
+
test failure.
|
|
105
|
+
|
|
106
|
+
## License
|
|
107
|
+
|
|
108
|
+
MIT
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "sheets-data-validation-kit"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Validation, reconciliation and entity-resolution primitives for spreadsheet and CSV pipelines"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 4 - Beta",
|
|
14
|
+
"Intended Audience :: Developers",
|
|
15
|
+
"License :: OSI Approved :: MIT License",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3.9",
|
|
18
|
+
"Programming Language :: Python :: 3.10",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Programming Language :: Python :: 3.14",
|
|
23
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
24
|
+
]
|
|
25
|
+
# No dependencies, on purpose: the standard library is the whole stack.
|
|
26
|
+
dependencies = []
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
dev = ["pytest>=8.0", "ruff>=0.6"]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/devops-gm88/sheets-data-validation-kit"
|
|
33
|
+
|
|
34
|
+
[tool.setuptools.packages.find]
|
|
35
|
+
where = ["src"]
|
|
36
|
+
|
|
37
|
+
[tool.ruff]
|
|
38
|
+
target-version = "py39"
|
|
39
|
+
line-length = 110
|
|
40
|
+
|
|
41
|
+
[tool.ruff.lint]
|
|
42
|
+
# UP is deliberately not selected: the source keeps `typing.Optional` / `List`
|
|
43
|
+
# annotations on purpose, and rewriting them would be churn with no behaviour change.
|
|
44
|
+
select = ["E", "F", "W", "I", "B"]
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sheets-data-validation-kit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Validation, reconciliation and entity-resolution primitives for spreadsheet and CSV pipelines
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/devops-gm88/sheets-data-validation-kit
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
18
|
+
Requires-Python: >=3.9
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
23
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# sheets-data-validation-kit
|
|
27
|
+
|
|
28
|
+
[](https://github.com/devops-gm88/sheets-data-validation-kit/actions/workflows/ci.yml)
|
|
29
|
+
[](pyproject.toml)
|
|
30
|
+
[](LICENSE)
|
|
31
|
+
|
|
32
|
+
Validation, reconciliation and entity-resolution primitives for spreadsheet and CSV pipelines.
|
|
33
|
+
|
|
34
|
+
Most data pipelines fail quietly. A row lands, a number looks plausible, and nobody notices it is wrong
|
|
35
|
+
until it has been reported to someone. This kit exists to make that failure mode loud.
|
|
36
|
+
|
|
37
|
+
It is deliberately **standard-library only** — no dependencies, nothing to install, nothing to upgrade.
|
|
38
|
+
Copy the `src/validation_kit` folder into your project, or install the package in one line:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install git+https://github.com/devops-gm88/sheets-data-validation-kit.git
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Not on PyPI yet - the GitHub URL above is the install. From a clone:
|
|
45
|
+
`pip install -e .`.
|
|
46
|
+
|
|
47
|
+
## The three ideas
|
|
48
|
+
|
|
49
|
+
**1. Rows pass or they are quarantined. They are never silently dropped.**
|
|
50
|
+
A row that fails a contract goes to quarantine with the reasons attached. You can always answer
|
|
51
|
+
"where did this record go?" — which is the question that kills trust in a pipeline.
|
|
52
|
+
|
|
53
|
+
**2. Arithmetic is a second, independent check.**
|
|
54
|
+
Field-level validation confirms each value looks right. Reconciliation confirms the values are
|
|
55
|
+
*consistent with each other*. An extracted balance sheet must balance; a cash flow statement must tie to
|
|
56
|
+
the change in cash. If the arithmetic does not close, the extraction failed regardless of how confident
|
|
57
|
+
each individual field was. This catches the failure nobody sees: clean extraction, quietly wrong numbers.
|
|
58
|
+
|
|
59
|
+
**3. Exact match or human review. Never a fuzzy merge.**
|
|
60
|
+
Duplicates are detected on a canonical key. Suspected duplicates are queued for a person, because a
|
|
61
|
+
silent merge corrupts counts invisibly and is very hard to unpick later.
|
|
62
|
+
|
|
63
|
+
## Quick start
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from validation_kit import RowContract, Field, Min, Range, Pattern, Reconcile, Assertion
|
|
67
|
+
from validation_kit.entities import resolve
|
|
68
|
+
|
|
69
|
+
contract = RowContract([
|
|
70
|
+
Field("account_id", required=True),
|
|
71
|
+
Field("amount", rules=[Min(0)], coerce=float,
|
|
72
|
+
on_fail="quarantine"),
|
|
73
|
+
Field("period", rules=[Pattern(r"^\d{4}-Q[1-4]$")]),
|
|
74
|
+
])
|
|
75
|
+
|
|
76
|
+
result = contract.validate_rows(rows)
|
|
77
|
+
print(result.summary())
|
|
78
|
+
for row, reasons in result.quarantined:
|
|
79
|
+
print(row.get("account_id"), reasons)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
# Arithmetic self-consistency: the check that does not depend on the model
|
|
84
|
+
suite = Reconcile([
|
|
85
|
+
Assertion("balance_sheet_balances",
|
|
86
|
+
lambda r: r["total_assets"], lambda r: r["total_liabilities"] + r["total_equity"],
|
|
87
|
+
tolerance=0.01),
|
|
88
|
+
])
|
|
89
|
+
|
|
90
|
+
for outcome in suite.check(parsed_statement):
|
|
91
|
+
print(outcome) # PASS / FAIL with both sides of the equation printed
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
# Entity resolution: exact key, review queue, no auto-merge
|
|
96
|
+
outcome = resolve(records, key="child_id")
|
|
97
|
+
print(outcome.unique_count, len(outcome.duplicate_candidates))
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## What this is not
|
|
101
|
+
|
|
102
|
+
- It is not a data-quality *scoring* tool. There is no 0-100 number and no dashboard.
|
|
103
|
+
- It does not fix your source data. It tells you which records are untrustworthy and why.
|
|
104
|
+
- It will not merge duplicates for you. That is the point.
|
|
105
|
+
|
|
106
|
+
## Why it exists
|
|
107
|
+
|
|
108
|
+
Built from repeatedly solving the same problem on real reporting systems: a monthly pack that was
|
|
109
|
+
*almost* right, and no way to prove which numbers were which. These are the primitives that were
|
|
110
|
+
written more than once, extracted so they are not written again.
|
|
111
|
+
|
|
112
|
+
## Tests
|
|
113
|
+
|
|
114
|
+
Install first, then run:
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
pip install -e .
|
|
118
|
+
python3 -m unittest discover -s tests -v
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Or without installing anything:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Both give **24 tests, OK**. Running `unittest` from a fresh clone without one of
|
|
128
|
+
those two steps fails to import the package — that is a packaging path, not a
|
|
129
|
+
test failure.
|
|
130
|
+
|
|
131
|
+
## License
|
|
132
|
+
|
|
133
|
+
MIT
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/sheets_data_validation_kit.egg-info/PKG-INFO
|
|
5
|
+
src/sheets_data_validation_kit.egg-info/SOURCES.txt
|
|
6
|
+
src/sheets_data_validation_kit.egg-info/dependency_links.txt
|
|
7
|
+
src/sheets_data_validation_kit.egg-info/requires.txt
|
|
8
|
+
src/sheets_data_validation_kit.egg-info/top_level.txt
|
|
9
|
+
src/validation_kit/__init__.py
|
|
10
|
+
src/validation_kit/contract.py
|
|
11
|
+
src/validation_kit/entities.py
|
|
12
|
+
src/validation_kit/reconcile.py
|
|
13
|
+
tests/test_contract.py
|
|
14
|
+
tests/test_entities.py
|
|
15
|
+
tests/test_reconcile.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
validation_kit
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Validation, reconciliation and entity-resolution primitives.
|
|
2
|
+
|
|
3
|
+
Standard library only, deliberately.
|
|
4
|
+
"""
|
|
5
|
+
from .contract import Field, Max, Min, NotBlank, OneOf, Pattern, Range, RowContract, ValidationResult
|
|
6
|
+
from .entities import ResolutionOutcome, resolve
|
|
7
|
+
from .reconcile import Assertion, Outcome, Reconcile
|
|
8
|
+
|
|
9
|
+
__version__ = "0.1.0"
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"Field", "RowContract", "ValidationResult",
|
|
13
|
+
"Min", "Max", "Range", "Pattern", "OneOf", "NotBlank",
|
|
14
|
+
"Assertion", "Reconcile", "Outcome",
|
|
15
|
+
"resolve", "ResolutionOutcome",
|
|
16
|
+
]
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Field-level contracts.
|
|
2
|
+
|
|
3
|
+
A row either satisfies its contract or it is quarantined with the reasons attached.
|
|
4
|
+
Quarantine is the default failure mode: nothing is ever silently dropped.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Rule:
|
|
13
|
+
"""Base class for a single field rule."""
|
|
14
|
+
|
|
15
|
+
name = "rule"
|
|
16
|
+
|
|
17
|
+
def check(self, value: Any) -> Optional[str]:
|
|
18
|
+
"""Return None if the value passes, else a human-readable reason."""
|
|
19
|
+
raise NotImplementedError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class Min(Rule):
|
|
24
|
+
bound: float
|
|
25
|
+
name = "min"
|
|
26
|
+
|
|
27
|
+
def check(self, value):
|
|
28
|
+
try:
|
|
29
|
+
return None if float(value) >= self.bound else f"below minimum {self.bound}"
|
|
30
|
+
except (TypeError, ValueError):
|
|
31
|
+
return f"not numeric, cannot compare to minimum {self.bound}"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class Max(Rule):
|
|
36
|
+
bound: float
|
|
37
|
+
name = "max"
|
|
38
|
+
|
|
39
|
+
def check(self, value):
|
|
40
|
+
try:
|
|
41
|
+
return None if float(value) <= self.bound else f"above maximum {self.bound}"
|
|
42
|
+
except (TypeError, ValueError):
|
|
43
|
+
return f"not numeric, cannot compare to maximum {self.bound}"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class Range(Rule):
|
|
48
|
+
low: float
|
|
49
|
+
high: float
|
|
50
|
+
name = "range"
|
|
51
|
+
|
|
52
|
+
def check(self, value):
|
|
53
|
+
try:
|
|
54
|
+
number = float(value)
|
|
55
|
+
except (TypeError, ValueError):
|
|
56
|
+
return f"not numeric, cannot range-check {self.low}..{self.high}"
|
|
57
|
+
if number < self.low or number > self.high:
|
|
58
|
+
return f"{number} outside plausible range {self.low}..{self.high}"
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class Pattern(Rule):
|
|
64
|
+
regex: str
|
|
65
|
+
name = "pattern"
|
|
66
|
+
|
|
67
|
+
def check(self, value):
|
|
68
|
+
import re
|
|
69
|
+
|
|
70
|
+
return None if re.match(self.regex, str(value)) else f"does not match {self.regex}"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass
|
|
74
|
+
class OneOf(Rule):
|
|
75
|
+
allowed: Sequence[Any]
|
|
76
|
+
name = "one_of"
|
|
77
|
+
|
|
78
|
+
def check(self, value):
|
|
79
|
+
return None if value in self.allowed else f"{value!r} not one of {list(self.allowed)}"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class NotBlank(Rule):
|
|
83
|
+
name = "not_blank"
|
|
84
|
+
|
|
85
|
+
def check(self, value):
|
|
86
|
+
if value is None or (isinstance(value, str) and not value.strip()):
|
|
87
|
+
return "blank"
|
|
88
|
+
return None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass
|
|
92
|
+
class Field:
|
|
93
|
+
"""One column in a row contract.
|
|
94
|
+
|
|
95
|
+
on_fail:
|
|
96
|
+
"quarantine" (default) - hold the whole row back for review.
|
|
97
|
+
"flag" - record the problem but keep the row.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
name: str
|
|
101
|
+
required: bool = False
|
|
102
|
+
rules: List[Rule] = field(default_factory=list)
|
|
103
|
+
coerce: Optional[Callable[[Any], Any]] = None
|
|
104
|
+
on_fail: str = "quarantine"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass
|
|
108
|
+
class ValidationResult:
|
|
109
|
+
accepted: List[Dict[str, Any]] = field(default_factory=list)
|
|
110
|
+
quarantined: List[Tuple[Dict[str, Any], List[str]]] = field(default_factory=list)
|
|
111
|
+
flagged: List[Tuple[Dict[str, Any], List[str]]] = field(default_factory=list)
|
|
112
|
+
field_error_counts: Dict[str, int] = field(default_factory=dict)
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def total(self) -> int:
|
|
116
|
+
return len(self.accepted) + len(self.quarantined)
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def pass_rate(self) -> float:
|
|
120
|
+
return 0.0 if self.total == 0 else len(self.accepted) / self.total
|
|
121
|
+
|
|
122
|
+
def summary(self) -> str:
|
|
123
|
+
lines = [
|
|
124
|
+
f"rows in : {self.total}",
|
|
125
|
+
f"accepted : {len(self.accepted)} ({self.pass_rate:.1%})",
|
|
126
|
+
f"quarantined : {len(self.quarantined)}",
|
|
127
|
+
f"flagged : {len(self.flagged)}",
|
|
128
|
+
]
|
|
129
|
+
if self.field_error_counts:
|
|
130
|
+
lines.append("errors by field:")
|
|
131
|
+
for name, count in sorted(self.field_error_counts.items(), key=lambda kv: -kv[1]):
|
|
132
|
+
lines.append(f" {name:<24} {count}")
|
|
133
|
+
return "\n".join(lines)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class RowContract:
|
|
137
|
+
"""A set of field rules applied to every row.
|
|
138
|
+
|
|
139
|
+
Rows that fail a quarantine field are held back. Rows that fail a flag field are
|
|
140
|
+
kept, with the problem recorded, so a pipeline can decide for itself how strict to be
|
|
141
|
+
on fields it does not depend on.
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
def __init__(self, fields: Sequence[Field]):
|
|
145
|
+
self.fields = list(fields)
|
|
146
|
+
|
|
147
|
+
def validate_row(self, row: Dict[str, Any]):
|
|
148
|
+
"""Returns (cleaned_row, reasons, verdict) where verdict is pass|quarantine|flag."""
|
|
149
|
+
cleaned: Dict[str, Any] = dict(row)
|
|
150
|
+
reasons: List[str] = []
|
|
151
|
+
severity = "pass"
|
|
152
|
+
|
|
153
|
+
for spec in self.fields:
|
|
154
|
+
value = row.get(spec.name)
|
|
155
|
+
|
|
156
|
+
if spec.required and (value is None or (isinstance(value, str) and not value.strip())):
|
|
157
|
+
reasons.append(f"{spec.name}: required but blank")
|
|
158
|
+
severity = "quarantine"
|
|
159
|
+
continue
|
|
160
|
+
|
|
161
|
+
if value is None or (isinstance(value, str) and not value.strip()):
|
|
162
|
+
continue
|
|
163
|
+
|
|
164
|
+
if spec.coerce is not None:
|
|
165
|
+
try:
|
|
166
|
+
value = spec.coerce(value)
|
|
167
|
+
cleaned[spec.name] = value
|
|
168
|
+
except (TypeError, ValueError) as exc:
|
|
169
|
+
reasons.append(f"{spec.name}: cannot coerce ({exc})")
|
|
170
|
+
severity = "quarantine" if spec.on_fail == "quarantine" else "flag"
|
|
171
|
+
continue
|
|
172
|
+
|
|
173
|
+
for rule in spec.rules:
|
|
174
|
+
problem = rule.check(value)
|
|
175
|
+
if problem:
|
|
176
|
+
reasons.append(f"{spec.name}: {problem}")
|
|
177
|
+
if spec.on_fail == "quarantine":
|
|
178
|
+
severity = "quarantine"
|
|
179
|
+
elif severity == "pass":
|
|
180
|
+
severity = "flag"
|
|
181
|
+
|
|
182
|
+
return cleaned, reasons, severity
|
|
183
|
+
|
|
184
|
+
def validate_rows(self, rows: Iterable[Dict[str, Any]]) -> ValidationResult:
|
|
185
|
+
result = ValidationResult()
|
|
186
|
+
for row in rows:
|
|
187
|
+
cleaned, reasons, verdict = self.validate_row(row)
|
|
188
|
+
if verdict == "pass":
|
|
189
|
+
result.accepted.append(cleaned)
|
|
190
|
+
elif verdict == "quarantine":
|
|
191
|
+
result.quarantined.append((cleaned, reasons))
|
|
192
|
+
_bump(result.field_error_counts, reasons)
|
|
193
|
+
else:
|
|
194
|
+
result.flagged.append((cleaned, reasons))
|
|
195
|
+
_bump(result.field_error_counts, reasons)
|
|
196
|
+
return result
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _bump(counter: Dict[str, int], reasons: Sequence[str]) -> None:
|
|
200
|
+
for reason in reasons:
|
|
201
|
+
counter[reason.split(":", 1)[0]] = counter.get(reason.split(":", 1)[0], 0) + 1
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Exact-match entity resolution with a human review queue.
|
|
2
|
+
|
|
3
|
+
The rule this module enforces: **never merge automatically.**
|
|
4
|
+
|
|
5
|
+
Name-based and fuzzy matching is the usual cause of inflated counts in reporting
|
|
6
|
+
pipelines, because common names collide and re-registrations create a second record.
|
|
7
|
+
Suspects are surfaced for a person to decide, not resolved by a heuristic.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections import defaultdict
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class DuplicateCandidate:
|
|
18
|
+
"""Two or more records that look like the same real entity but carry different keys."""
|
|
19
|
+
|
|
20
|
+
reason: str
|
|
21
|
+
records: List[Dict[str, Any]]
|
|
22
|
+
|
|
23
|
+
def keys(self) -> List[Any]:
|
|
24
|
+
return [r.get("_key") for r in self.records]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class ResolutionOutcome:
|
|
29
|
+
unique: List[Dict[str, Any]] = field(default_factory=list)
|
|
30
|
+
duplicate_keys: List[Tuple[Any, int]] = field(default_factory=list)
|
|
31
|
+
duplicate_candidates: List[DuplicateCandidate] = field(default_factory=list)
|
|
32
|
+
blank_keys: List[Dict[str, Any]] = field(default_factory=list)
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def unique_count(self) -> int:
|
|
36
|
+
return len(self.unique)
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def input_count(self) -> int:
|
|
40
|
+
return self.unique_count + sum(count - 1 for _, count in self.duplicate_keys)
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def inflation(self) -> int:
|
|
44
|
+
"""How many phantom entities a naive row count would have reported."""
|
|
45
|
+
return sum(count - 1 for _, count in self.duplicate_keys)
|
|
46
|
+
|
|
47
|
+
def summary(self) -> str:
|
|
48
|
+
return (
|
|
49
|
+
f"records in : {self.input_count}\n"
|
|
50
|
+
f"distinct entities : {self.unique_count}\n"
|
|
51
|
+
f"phantom entities : {self.inflation}\n"
|
|
52
|
+
f"blank keys held back : {len(self.blank_keys)}\n"
|
|
53
|
+
f"suspects for review : {len(self.duplicate_candidates)}"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def resolve(records: Iterable[Dict[str, Any]],
|
|
58
|
+
key: str,
|
|
59
|
+
signature: Optional[Sequence[str]] = None,
|
|
60
|
+
normalise: Optional[Callable[[Any], Any]] = None) -> ResolutionOutcome:
|
|
61
|
+
"""Resolve records to distinct entities on an exact key.
|
|
62
|
+
|
|
63
|
+
key the canonical identifier. Rows without one are held back, never counted.
|
|
64
|
+
signature optional tuple of fields used only to *suggest* duplicates. Suggestions
|
|
65
|
+
are queued for review; they are never merged.
|
|
66
|
+
normalise optional callable applied to the key (strip/casefold and similar).
|
|
67
|
+
"""
|
|
68
|
+
outcome = ResolutionOutcome()
|
|
69
|
+
norm = normalise or (lambda v: str(v).strip().casefold() if v is not None else None)
|
|
70
|
+
|
|
71
|
+
buckets: Dict[Any, List[Dict[str, Any]]] = defaultdict(list)
|
|
72
|
+
for record in records:
|
|
73
|
+
raw = record.get(key)
|
|
74
|
+
if raw is None or (isinstance(raw, str) and not raw.strip()):
|
|
75
|
+
outcome.blank_keys.append(record)
|
|
76
|
+
continue
|
|
77
|
+
record = dict(record)
|
|
78
|
+
record["_key"] = norm(raw)
|
|
79
|
+
buckets[record["_key"]].append(record)
|
|
80
|
+
|
|
81
|
+
for entity_key, group in buckets.items():
|
|
82
|
+
group_sorted = sorted(group, key=lambda r: str(r.get("_source", "")))
|
|
83
|
+
outcome.unique.append(group_sorted[0])
|
|
84
|
+
if len(group) > 1:
|
|
85
|
+
outcome.duplicate_keys.append((entity_key, len(group)))
|
|
86
|
+
|
|
87
|
+
outcome.duplicate_keys.sort(key=lambda kv: -kv[1])
|
|
88
|
+
|
|
89
|
+
if signature:
|
|
90
|
+
seen: Dict[Tuple[Any, ...], List[Dict[str, Any]]] = defaultdict(list)
|
|
91
|
+
for record in outcome.unique:
|
|
92
|
+
token = tuple(norm(record.get(f)) for f in signature)
|
|
93
|
+
if any(part is None for part in token):
|
|
94
|
+
continue
|
|
95
|
+
seen[token].append(record)
|
|
96
|
+
for token, group in seen.items():
|
|
97
|
+
if len(group) > 1:
|
|
98
|
+
outcome.duplicate_candidates.append(
|
|
99
|
+
DuplicateCandidate(
|
|
100
|
+
reason=f"same {'/'.join(signature)}: {token}",
|
|
101
|
+
records=group,
|
|
102
|
+
)
|
|
103
|
+
)
|
|
104
|
+
outcome.duplicate_candidates.sort(key=lambda c: -len(c.records))
|
|
105
|
+
|
|
106
|
+
return outcome
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def count_distinct(records: Iterable[Dict[str, Any]], key: str) -> int:
|
|
110
|
+
"""Distinct entity count. Blank keys are excluded rather than counted as one entity."""
|
|
111
|
+
values = {str(r.get(key)).strip().casefold() for r in records
|
|
112
|
+
if r.get(key) is not None and str(r.get(key)).strip()}
|
|
113
|
+
return len(values)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Arithmetic reconciliation.
|
|
2
|
+
|
|
3
|
+
Field-level validation asks "does this value look right?".
|
|
4
|
+
Reconciliation asks "are these values consistent with each other?".
|
|
5
|
+
|
|
6
|
+
It is a genuinely independent check: it needs no model, no confidence score and no
|
|
7
|
+
reference data, only arithmetic that must hold. If a statement extracts cleanly and the
|
|
8
|
+
arithmetic does not close, the extraction failed.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import Any, Callable, Dict, List, Sequence
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class Outcome:
|
|
18
|
+
name: str
|
|
19
|
+
passed: bool
|
|
20
|
+
left: float
|
|
21
|
+
right: float
|
|
22
|
+
difference: float
|
|
23
|
+
tolerance: float
|
|
24
|
+
note: str = ""
|
|
25
|
+
|
|
26
|
+
def __str__(self) -> str:
|
|
27
|
+
mark = "PASS" if self.passed else "FAIL"
|
|
28
|
+
detail = f"{self.left:,.2f} vs {self.right:,.2f} (diff {self.difference:,.4f}, tol {self.tolerance})"
|
|
29
|
+
return f"[{mark}] {self.name}: {detail}" + (f" -- {self.note}" if self.note else "")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Assertion:
|
|
34
|
+
"""One equation that must hold for a record.
|
|
35
|
+
|
|
36
|
+
left and right are callables taking the record and returning a number, so the
|
|
37
|
+
assertion reads like the equation it tests:
|
|
38
|
+
|
|
39
|
+
Assertion("assets = liabilities + equity",
|
|
40
|
+
lambda r: r["total_assets"],
|
|
41
|
+
lambda r: r["total_liabilities"] + r["total_equity"])
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
name: str
|
|
45
|
+
left: Callable[[Dict[str, Any]], float]
|
|
46
|
+
right: Callable[[Dict[str, Any]], float]
|
|
47
|
+
tolerance: float = 0.01
|
|
48
|
+
note: str = ""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class Reconcile:
|
|
52
|
+
def __init__(self, assertions: Sequence[Assertion]):
|
|
53
|
+
self.assertions = list(assertions)
|
|
54
|
+
|
|
55
|
+
def check(self, record: Dict[str, Any]) -> List[Outcome]:
|
|
56
|
+
outcomes: List[Outcome] = []
|
|
57
|
+
for assertion in self.assertions:
|
|
58
|
+
try:
|
|
59
|
+
left = float(assertion.left(record))
|
|
60
|
+
right = float(assertion.right(record))
|
|
61
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
62
|
+
outcomes.append(Outcome(assertion.name, False, float("nan"), float("nan"),
|
|
63
|
+
float("nan"), assertion.tolerance,
|
|
64
|
+
note=f"cannot evaluate: {exc}"))
|
|
65
|
+
continue
|
|
66
|
+
difference = abs(left - right)
|
|
67
|
+
outcomes.append(Outcome(assertion.name, difference <= assertion.tolerance,
|
|
68
|
+
left, right, difference, assertion.tolerance, assertion.note))
|
|
69
|
+
return outcomes
|
|
70
|
+
|
|
71
|
+
def all_pass(self, record: Dict[str, Any]) -> bool:
|
|
72
|
+
return all(o.passed for o in self.check(record))
|
|
73
|
+
|
|
74
|
+
def failures(self, record: Dict[str, Any]) -> List[Outcome]:
|
|
75
|
+
return [o for o in self.check(record) if not o.passed]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def closing_balance_check(opening: str, inflow: str, outflow: str, closing: str,
|
|
79
|
+
tolerance: float = 0.01) -> Assertion:
|
|
80
|
+
"""closing == opening + inflow - outflow."""
|
|
81
|
+
return Assertion(
|
|
82
|
+
"closing = opening + inflow - outflow",
|
|
83
|
+
lambda r: r[closing],
|
|
84
|
+
lambda r: r[opening] + r[inflow] - r[outflow],
|
|
85
|
+
tolerance=tolerance,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def rollforward_check(opening: str, additions: str, reductions: str, closing: str,
|
|
90
|
+
tolerance: float = 0.01) -> Assertion:
|
|
91
|
+
"""closing == opening + additions - reductions. Used for inventory, PP&E, capital assets."""
|
|
92
|
+
return Assertion(
|
|
93
|
+
"closing = opening + additions - reductions",
|
|
94
|
+
lambda r: r[closing],
|
|
95
|
+
lambda r: r[opening] + r[additions] - r[reductions],
|
|
96
|
+
tolerance=tolerance,
|
|
97
|
+
)
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
|
|
3
|
+
from validation_kit import Field, Min, NotBlank, OneOf, Pattern, Range, RowContract
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TestRowContract(unittest.TestCase):
|
|
7
|
+
def setUp(self):
|
|
8
|
+
self.contract = RowContract([
|
|
9
|
+
Field("account_id", required=True),
|
|
10
|
+
Field("amount", rules=[Min(0)], coerce=float),
|
|
11
|
+
Field("period", rules=[Pattern(r"^\d{4}-Q[1-4]$")]),
|
|
12
|
+
])
|
|
13
|
+
|
|
14
|
+
def test_clean_row_is_accepted(self):
|
|
15
|
+
result = self.contract.validate_rows(
|
|
16
|
+
[{"account_id": "A1", "amount": "100.50", "period": "2026-Q1"}])
|
|
17
|
+
self.assertEqual(len(result.accepted), 1)
|
|
18
|
+
self.assertEqual(result.accepted[0]["amount"], 100.5)
|
|
19
|
+
self.assertEqual(result.quarantined, [])
|
|
20
|
+
|
|
21
|
+
def test_blank_required_field_is_quarantined_not_dropped(self):
|
|
22
|
+
result = self.contract.validate_rows([{"account_id": " ", "amount": "10"}])
|
|
23
|
+
self.assertEqual(result.accepted, [])
|
|
24
|
+
self.assertEqual(len(result.quarantined), 1)
|
|
25
|
+
self.assertIn("required but blank", result.quarantined[0][1][0])
|
|
26
|
+
|
|
27
|
+
def test_out_of_range_amount_is_quarantined(self):
|
|
28
|
+
result = self.contract.validate_rows(
|
|
29
|
+
[{"account_id": "A2", "amount": "-5", "period": "2026-Q2"}])
|
|
30
|
+
self.assertEqual(len(result.quarantined), 1)
|
|
31
|
+
self.assertIn("below minimum", result.quarantined[0][1][0])
|
|
32
|
+
|
|
33
|
+
def test_malformed_period_is_quarantined(self):
|
|
34
|
+
result = self.contract.validate_rows(
|
|
35
|
+
[{"account_id": "A3", "amount": "5", "period": "Q1-2026"}])
|
|
36
|
+
self.assertEqual(len(result.quarantined), 1)
|
|
37
|
+
|
|
38
|
+
def test_nothing_is_ever_silently_lost(self):
|
|
39
|
+
rows = [
|
|
40
|
+
{"account_id": "A1", "amount": "1", "period": "2026-Q1"},
|
|
41
|
+
{"account_id": "", "amount": "1", "period": "2026-Q1"},
|
|
42
|
+
{"account_id": "A3", "amount": "x", "period": "2026-Q1"},
|
|
43
|
+
]
|
|
44
|
+
result = self.contract.validate_rows(rows)
|
|
45
|
+
self.assertEqual(result.total, len(rows))
|
|
46
|
+
|
|
47
|
+
def test_flag_mode_keeps_the_row(self):
|
|
48
|
+
lenient = RowContract([Field("note", rules=[NotBlank()], on_fail="flag")])
|
|
49
|
+
result = lenient.validate_rows([{"note": ""}, {"note": "ok"}])
|
|
50
|
+
self.assertEqual(len(result.accepted), 2)
|
|
51
|
+
|
|
52
|
+
def test_error_counts_are_reported_per_field(self):
|
|
53
|
+
result = self.contract.validate_rows([
|
|
54
|
+
{"account_id": "", "amount": "1", "period": "2026-Q1"},
|
|
55
|
+
{"account_id": "", "amount": "1", "period": "2026-Q1"},
|
|
56
|
+
])
|
|
57
|
+
self.assertEqual(result.field_error_counts["account_id"], 2)
|
|
58
|
+
|
|
59
|
+
def test_summary_reports_pass_rate(self):
|
|
60
|
+
result = self.contract.validate_rows([
|
|
61
|
+
{"account_id": "A1", "amount": "1", "period": "2026-Q1"},
|
|
62
|
+
{"account_id": "", "amount": "1", "period": "2026-Q1"},
|
|
63
|
+
])
|
|
64
|
+
self.assertIn("50.0%", result.summary())
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class TestRules(unittest.TestCase):
|
|
68
|
+
def test_one_of(self):
|
|
69
|
+
contract = RowContract([Field("stage", rules=[OneOf(["open", "closed"])])])
|
|
70
|
+
self.assertEqual(len(contract.validate_rows([{"stage": "open"}]).accepted), 1)
|
|
71
|
+
self.assertEqual(len(contract.validate_rows([{"stage": "pending"}]).quarantined), 1)
|
|
72
|
+
|
|
73
|
+
def test_range_catches_outliers(self):
|
|
74
|
+
contract = RowContract([Field("rate", rules=[Range(0.5, 1.5)], coerce=float)])
|
|
75
|
+
self.assertEqual(len(contract.validate_rows([{"rate": "1.2"}]).accepted), 1)
|
|
76
|
+
self.assertEqual(len(contract.validate_rows([{"rate": "12"}]).quarantined), 1)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
if __name__ == "__main__":
|
|
80
|
+
unittest.main()
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
|
|
3
|
+
from validation_kit.entities import count_distinct, resolve
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TestResolve(unittest.TestCase):
|
|
7
|
+
def test_distinct_keys_counted_once(self):
|
|
8
|
+
outcome = resolve([{"child_id": "C1"}, {"child_id": "C2"}], key="child_id")
|
|
9
|
+
self.assertEqual(outcome.unique_count, 2)
|
|
10
|
+
self.assertEqual(outcome.inflation, 0)
|
|
11
|
+
|
|
12
|
+
def test_repeat_key_does_not_inflate_the_count(self):
|
|
13
|
+
outcome = resolve([{"child_id": "C1"}, {"child_id": "C1"}, {"child_id": "C2"}],
|
|
14
|
+
key="child_id")
|
|
15
|
+
self.assertEqual(outcome.unique_count, 2)
|
|
16
|
+
self.assertEqual(outcome.inflation, 1)
|
|
17
|
+
|
|
18
|
+
def test_blank_keys_are_held_back_not_counted(self):
|
|
19
|
+
outcome = resolve([{"child_id": ""}, {"child_id": None}, {"child_id": "C1"}],
|
|
20
|
+
key="child_id")
|
|
21
|
+
self.assertEqual(outcome.unique_count, 1)
|
|
22
|
+
self.assertEqual(len(outcome.blank_keys), 2)
|
|
23
|
+
|
|
24
|
+
def test_case_and_whitespace_are_normalised(self):
|
|
25
|
+
outcome = resolve([{"child_id": " c1 "}, {"child_id": "C1"}], key="child_id")
|
|
26
|
+
self.assertEqual(outcome.unique_count, 1)
|
|
27
|
+
|
|
28
|
+
def test_suspects_are_queued_never_merged(self):
|
|
29
|
+
outcome = resolve([
|
|
30
|
+
{"child_id": "C1", "full_name": "Jane Doe"},
|
|
31
|
+
{"child_id": "C2", "full_name": "Jane Doe"},
|
|
32
|
+
], key="child_id", signature=["full_name"])
|
|
33
|
+
self.assertEqual(outcome.unique_count, 2, "suspects must not be merged")
|
|
34
|
+
self.assertEqual(len(outcome.duplicate_candidates), 1)
|
|
35
|
+
self.assertEqual(outcome.duplicate_candidates[0].keys(), ["c1", "c2"])
|
|
36
|
+
|
|
37
|
+
def test_inflation_reports_the_phantom_entities(self):
|
|
38
|
+
outcome = resolve([{"child_id": "C1"}] * 5, key="child_id")
|
|
39
|
+
self.assertEqual(outcome.unique_count, 1)
|
|
40
|
+
self.assertEqual(outcome.inflation, 4)
|
|
41
|
+
|
|
42
|
+
def test_count_distinct_helper(self):
|
|
43
|
+
self.assertEqual(count_distinct([{"k": "a"}, {"k": "a"}, {"k": ""}], "k"), 1)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
if __name__ == "__main__":
|
|
47
|
+
unittest.main()
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
|
|
3
|
+
from validation_kit import Assertion, Reconcile
|
|
4
|
+
from validation_kit.reconcile import closing_balance_check, rollforward_check
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TestReconcile(unittest.TestCase):
|
|
8
|
+
def setUp(self):
|
|
9
|
+
self.suite = Reconcile([
|
|
10
|
+
Assertion("assets = liabilities + equity",
|
|
11
|
+
lambda r: r["total_assets"],
|
|
12
|
+
lambda r: r["total_liabilities"] + r["total_equity"],
|
|
13
|
+
tolerance=0.01),
|
|
14
|
+
])
|
|
15
|
+
|
|
16
|
+
def test_balanced_record_passes(self):
|
|
17
|
+
record = {"total_assets": 1000.0, "total_liabilities": 400.0, "total_equity": 600.0}
|
|
18
|
+
self.assertTrue(self.suite.all_pass(record))
|
|
19
|
+
|
|
20
|
+
def test_unbalanced_record_fails(self):
|
|
21
|
+
record = {"total_assets": 1000.0, "total_liabilities": 400.0, "total_equity": 599.0}
|
|
22
|
+
self.assertFalse(self.suite.all_pass(record))
|
|
23
|
+
failure = self.suite.failures(record)[0]
|
|
24
|
+
self.assertEqual(failure.left, 1000.0)
|
|
25
|
+
self.assertAlmostEqual(failure.difference, 1.0)
|
|
26
|
+
|
|
27
|
+
def test_tolerance_is_respected(self):
|
|
28
|
+
record = {"total_assets": 1000.005, "total_liabilities": 400.0, "total_equity": 600.0}
|
|
29
|
+
self.assertTrue(self.suite.all_pass(record))
|
|
30
|
+
|
|
31
|
+
def test_missing_field_is_a_failure_not_an_exception(self):
|
|
32
|
+
outcomes = self.suite.check({"total_assets": 1000.0})
|
|
33
|
+
self.assertFalse(outcomes[0].passed)
|
|
34
|
+
self.assertIn("cannot evaluate", outcomes[0].note)
|
|
35
|
+
|
|
36
|
+
def test_outcome_string_prints_both_sides(self):
|
|
37
|
+
record = {"total_assets": 1000.0, "total_liabilities": 400.0, "total_equity": 500.0}
|
|
38
|
+
text = str(self.suite.check(record)[0])
|
|
39
|
+
self.assertIn("FAIL", text)
|
|
40
|
+
self.assertIn("1,000.00", text)
|
|
41
|
+
|
|
42
|
+
def test_closing_balance_helper(self):
|
|
43
|
+
suite = Reconcile([closing_balance_check("opening", "inflow", "outflow", "closing")])
|
|
44
|
+
self.assertTrue(suite.all_pass(
|
|
45
|
+
{"opening": 100, "inflow": 50, "outflow": 30, "closing": 120}))
|
|
46
|
+
self.assertFalse(suite.all_pass(
|
|
47
|
+
{"opening": 100, "inflow": 50, "outflow": 30, "closing": 119}))
|
|
48
|
+
|
|
49
|
+
def test_rollforward_helper(self):
|
|
50
|
+
suite = Reconcile([rollforward_check("opening", "additions", "reductions", "closing")])
|
|
51
|
+
self.assertTrue(suite.all_pass(
|
|
52
|
+
{"opening": 10, "additions": 5, "reductions": 2, "closing": 13}))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
if __name__ == "__main__":
|
|
56
|
+
unittest.main()
|