datacheckup 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datacheckup-0.1.0/LICENSE +21 -0
- datacheckup-0.1.0/PKG-INFO +64 -0
- datacheckup-0.1.0/README.md +44 -0
- datacheckup-0.1.0/datacheckup/__init__.py +25 -0
- datacheckup-0.1.0/datacheckup/checks.py +219 -0
- datacheckup-0.1.0/datacheckup/detector.py +91 -0
- datacheckup-0.1.0/datacheckup/errors.py +2 -0
- datacheckup-0.1.0/datacheckup/exporter.py +73 -0
- datacheckup-0.1.0/datacheckup/messages.py +121 -0
- datacheckup-0.1.0/datacheckup/reader.py +216 -0
- datacheckup-0.1.0/datacheckup/report.py +130 -0
- datacheckup-0.1.0/datacheckup/rules.py +166 -0
- datacheckup-0.1.0/datacheckup/settings.py +63 -0
- datacheckup-0.1.0/datacheckup.egg-info/PKG-INFO +64 -0
- datacheckup-0.1.0/datacheckup.egg-info/SOURCES.txt +20 -0
- datacheckup-0.1.0/datacheckup.egg-info/dependency_links.txt +1 -0
- datacheckup-0.1.0/datacheckup.egg-info/top_level.txt +1 -0
- datacheckup-0.1.0/pyproject.toml +28 -0
- datacheckup-0.1.0/setup.cfg +4 -0
- datacheckup-0.1.0/tests/test_compatibility.py +126 -0
- datacheckup-0.1.0/tests/test_datacheckup.py +540 -0
- datacheckup-0.1.0/tests/test_override.py +222 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 CoreFixGroup
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datacheckup
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Find the exact row and column of every error in a CSV or JSON file, with no setup and no database.
|
|
5
|
+
Author: Corefix Group
|
|
6
|
+
Maintainer: Corefix Group
|
|
7
|
+
License: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/CoreFixGroup/DataCheckup
|
|
9
|
+
Project-URL: Repository, https://github.com/CoreFixGroup/DataCheckup
|
|
10
|
+
Project-URL: Issues, https://github.com/CoreFixGroup/DataCheckup/issues
|
|
11
|
+
Keywords: data quality,csv,json,validation,data cleaning
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# datacheckup
|
|
22
|
+
|
|
23
|
+
A small Python library that reads a CSV or JSON file and tells you the exact row and column of every error, with a plain-words reason and a fix. It finds the rules for each column by itself, lets you override only the ones you choose, and never changes your original file. It needs nothing but the Python standard library.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
pip install datacheckup
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Use
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from datacheckup import check
|
|
35
|
+
|
|
36
|
+
report = check("students.csv")
|
|
37
|
+
report.show()
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Override rules for chosen columns
|
|
41
|
+
|
|
42
|
+
Name only the columns you want to change. Every other column stays automatic.
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from datacheckup import check
|
|
46
|
+
|
|
47
|
+
rules = {
|
|
48
|
+
"Marks": {"min": 10, "max": 80},
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
report = check("students.csv", rules=rules)
|
|
52
|
+
report.show()
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Good to know
|
|
56
|
+
|
|
57
|
+
- Rules found automatically are smart guesses. Use your own rules for columns that matter.
|
|
58
|
+
- The whole file is loaded into memory, so very large files need a lot of RAM.
|
|
59
|
+
- Excel (.xlsx) files must be saved as CSV first.
|
|
60
|
+
- Requires Python 3.11 or newer.
|
|
61
|
+
|
|
62
|
+
## Licence
|
|
63
|
+
|
|
64
|
+
MIT
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# datacheckup
|
|
2
|
+
|
|
3
|
+
A small Python library that reads a CSV or JSON file and tells you the exact row and column of every error, with a plain-words reason and a fix. It finds the rules for each column by itself, lets you override only the ones you choose, and never changes your original file. It needs nothing but the Python standard library.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
pip install datacheckup
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Use
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
from datacheckup import check
|
|
15
|
+
|
|
16
|
+
report = check("students.csv")
|
|
17
|
+
report.show()
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Override rules for chosen columns
|
|
21
|
+
|
|
22
|
+
Name only the columns you want to change. Every other column stays automatic.
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
from datacheckup import check
|
|
26
|
+
|
|
27
|
+
rules = {
|
|
28
|
+
"Marks": {"min": 10, "max": 80},
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
report = check("students.csv", rules=rules)
|
|
32
|
+
report.show()
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Good to know
|
|
36
|
+
|
|
37
|
+
- Rules found automatically are smart guesses. Use your own rules for columns that matter.
|
|
38
|
+
- The whole file is loaded into memory, so very large files need a lot of RAM.
|
|
39
|
+
- Excel (.xlsx) files must be saved as CSV first.
|
|
40
|
+
- Requires Python 3.11 or newer.
|
|
41
|
+
|
|
42
|
+
## Licence
|
|
43
|
+
|
|
44
|
+
MIT
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from .checks import run_checks
|
|
2
|
+
from .detector import detect_rules
|
|
3
|
+
from .errors import DataCheckupError
|
|
4
|
+
from .reader import read_file
|
|
5
|
+
from .report import Report
|
|
6
|
+
from .rules import combine_rules, load_rules
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def check(path, rules=None):
|
|
10
|
+
custom = load_rules(rules, partial=True) if rules is not None else {}
|
|
11
|
+
columns, records, problems = read_file(path)
|
|
12
|
+
detected, notes = detect_rules(columns, records)
|
|
13
|
+
final, custom_notes = combine_rules(detected, custom, columns)
|
|
14
|
+
errors = run_checks(columns, records, final, problems)
|
|
15
|
+
return Report(path, columns, records, errors, final, True, notes + custom_notes)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def check_with_rules(path, rules):
|
|
19
|
+
rules = load_rules(rules)
|
|
20
|
+
columns, records, problems = read_file(path)
|
|
21
|
+
errors = run_checks(columns, records, rules, problems)
|
|
22
|
+
return Report(path, columns, records, errors, rules, False, [])
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
__all__ = ["check", "Report", "DataCheckupError"]
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from . import messages, settings
|
|
4
|
+
from .rules import FORMAT_CHECKS, parse_number
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def is_empty(value):
|
|
8
|
+
return value is None or str(value).strip() == ""
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def mask_value(value):
|
|
12
|
+
text = str(value).strip()
|
|
13
|
+
if len(text) <= 4:
|
|
14
|
+
return "*" * len(text)
|
|
15
|
+
return text[:2] + "*" * (len(text) - 4) + text[-2:]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def display_value(value, rule):
|
|
19
|
+
if value is None:
|
|
20
|
+
return ""
|
|
21
|
+
text = str(value).strip()
|
|
22
|
+
if rule.get("sensitive") or rule.get("format") in settings.SENSITIVE_FORMATS:
|
|
23
|
+
return mask_value(text)
|
|
24
|
+
return text
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def make_error(row, column, value, error_type, reason, suggestion=""):
|
|
28
|
+
return {
|
|
29
|
+
"row": row,
|
|
30
|
+
"column": column,
|
|
31
|
+
"value": value,
|
|
32
|
+
"error_type": error_type,
|
|
33
|
+
"reason": reason,
|
|
34
|
+
"suggestion": suggestion,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def check_structure(problems):
|
|
39
|
+
return [
|
|
40
|
+
make_error(p["row"], p["column"], "", "structure", p["reason"], p["suggestion"])
|
|
41
|
+
for p in problems
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def check_schema(columns, rules):
|
|
46
|
+
errors = []
|
|
47
|
+
for column, rule in rules.items():
|
|
48
|
+
if column not in columns and rule.get("required"):
|
|
49
|
+
errors.append(make_error(
|
|
50
|
+
None, column, "", "schema",
|
|
51
|
+
messages.MISSING_COLUMN,
|
|
52
|
+
messages.MISSING_COLUMN_FIX.format(column=column),
|
|
53
|
+
))
|
|
54
|
+
return errors
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def check_missing(records, rules):
|
|
58
|
+
errors = []
|
|
59
|
+
for row, record in records:
|
|
60
|
+
for column, rule in rules.items():
|
|
61
|
+
if rule.get("required") and is_empty(record.get(column)):
|
|
62
|
+
errors.append(make_error(
|
|
63
|
+
row, column, "", "missing",
|
|
64
|
+
messages.MISSING_VALUE,
|
|
65
|
+
messages.MISSING_VALUE_FIX.format(column=column),
|
|
66
|
+
))
|
|
67
|
+
return errors
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def check_number(row, column, rule, value):
|
|
71
|
+
kind = rule["type"]
|
|
72
|
+
shown = display_value(value, rule)
|
|
73
|
+
number = parse_number(value, kind)
|
|
74
|
+
if number is None:
|
|
75
|
+
expected = messages.EXPECTED_WHOLE_NUMBER if kind == "integer" else messages.EXPECTED_NUMBER
|
|
76
|
+
return [make_error(
|
|
77
|
+
row, column, shown, "invalid_type",
|
|
78
|
+
messages.WRONG_TYPE.format(expected=expected),
|
|
79
|
+
messages.WRONG_TYPE_FIX,
|
|
80
|
+
)]
|
|
81
|
+
errors = []
|
|
82
|
+
if "min" in rule and number < rule["min"]:
|
|
83
|
+
errors.append(make_error(
|
|
84
|
+
row, column, shown, "out_of_range",
|
|
85
|
+
messages.BELOW_MINIMUM.format(limit=rule["min"]),
|
|
86
|
+
messages.BELOW_MINIMUM_FIX.format(limit=rule["min"]),
|
|
87
|
+
))
|
|
88
|
+
if "max" in rule and number > rule["max"]:
|
|
89
|
+
errors.append(make_error(
|
|
90
|
+
row, column, shown, "out_of_range",
|
|
91
|
+
messages.ABOVE_MAXIMUM.format(limit=rule["max"]),
|
|
92
|
+
messages.ABOVE_MAXIMUM_FIX.format(limit=rule["max"]),
|
|
93
|
+
))
|
|
94
|
+
return errors
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def check_types_and_ranges(records, rules):
|
|
98
|
+
errors = []
|
|
99
|
+
for row, record in records:
|
|
100
|
+
for column, rule in rules.items():
|
|
101
|
+
if rule.get("type") not in settings.NUMBER_TYPES:
|
|
102
|
+
continue
|
|
103
|
+
value = record.get(column)
|
|
104
|
+
if not is_empty(value):
|
|
105
|
+
errors += check_number(row, column, rule, value)
|
|
106
|
+
return errors
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def check_formats(records, rules):
|
|
110
|
+
errors = []
|
|
111
|
+
for row, record in records:
|
|
112
|
+
for column, rule in rules.items():
|
|
113
|
+
name = rule.get("format")
|
|
114
|
+
value = record.get(column)
|
|
115
|
+
if not name or is_empty(value):
|
|
116
|
+
continue
|
|
117
|
+
if not FORMAT_CHECKS[name](str(value).strip()):
|
|
118
|
+
errors.append(make_error(
|
|
119
|
+
row, column, display_value(value, rule), "invalid_format",
|
|
120
|
+
messages.WRONG_FORMAT.format(format=name),
|
|
121
|
+
messages.FORMAT_FIX[name],
|
|
122
|
+
))
|
|
123
|
+
return errors
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def check_patterns(records, rules):
|
|
127
|
+
errors = []
|
|
128
|
+
for column, rule in rules.items():
|
|
129
|
+
if "pattern" not in rule:
|
|
130
|
+
continue
|
|
131
|
+
pattern = re.compile(rule["pattern"])
|
|
132
|
+
reason = rule.get("message") or messages.PATTERN_MISMATCH.format(pattern=rule["pattern"])
|
|
133
|
+
for row, record in records:
|
|
134
|
+
value = record.get(column)
|
|
135
|
+
if is_empty(value) or pattern.fullmatch(str(value).strip()):
|
|
136
|
+
continue
|
|
137
|
+
errors.append(make_error(
|
|
138
|
+
row, column, display_value(value, rule), "invalid_format",
|
|
139
|
+
reason, messages.PATTERN_FIX,
|
|
140
|
+
))
|
|
141
|
+
return errors
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def check_allowed(records, rules):
|
|
145
|
+
errors = []
|
|
146
|
+
for column, rule in rules.items():
|
|
147
|
+
if "allowed" not in rule:
|
|
148
|
+
continue
|
|
149
|
+
allowed = [str(item).strip() for item in rule["allowed"]]
|
|
150
|
+
reason = rule.get("message") or messages.NOT_ALLOWED
|
|
151
|
+
fix = messages.NOT_ALLOWED_FIX.format(values=", ".join(allowed))
|
|
152
|
+
for row, record in records:
|
|
153
|
+
value = record.get(column)
|
|
154
|
+
if is_empty(value) or str(value).strip() in allowed:
|
|
155
|
+
continue
|
|
156
|
+
errors.append(make_error(
|
|
157
|
+
row, column, display_value(value, rule), "not_allowed", reason, fix,
|
|
158
|
+
))
|
|
159
|
+
return errors
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def check_duplicates(records, rules):
|
|
163
|
+
errors = []
|
|
164
|
+
for column, rule in rules.items():
|
|
165
|
+
if not rule.get("unique"):
|
|
166
|
+
continue
|
|
167
|
+
first_seen = {}
|
|
168
|
+
for row, record in records:
|
|
169
|
+
value = record.get(column)
|
|
170
|
+
if is_empty(value):
|
|
171
|
+
continue
|
|
172
|
+
key = str(value).strip()
|
|
173
|
+
if key in first_seen:
|
|
174
|
+
errors.append(make_error(
|
|
175
|
+
row, column, display_value(value, rule), "duplicate",
|
|
176
|
+
messages.DUPLICATE_VALUE.format(row=first_seen[key]),
|
|
177
|
+
messages.DUPLICATE_VALUE_FIX,
|
|
178
|
+
))
|
|
179
|
+
else:
|
|
180
|
+
first_seen[key] = row
|
|
181
|
+
return errors
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def row_key(columns, record):
|
|
185
|
+
return tuple("" if is_empty(record.get(c)) else str(record.get(c)).strip() for c in columns)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def check_duplicate_rows(columns, records):
|
|
189
|
+
errors = []
|
|
190
|
+
first_seen = {}
|
|
191
|
+
for row, record in records:
|
|
192
|
+
key = row_key(columns, record)
|
|
193
|
+
if not any(key):
|
|
194
|
+
continue
|
|
195
|
+
if key in first_seen:
|
|
196
|
+
errors.append(make_error(
|
|
197
|
+
row, settings.ENTIRE_ROW, "", "duplicate_row",
|
|
198
|
+
messages.DUPLICATE_ROW.format(row=first_seen[key]),
|
|
199
|
+
messages.DUPLICATE_ROW_FIX,
|
|
200
|
+
))
|
|
201
|
+
else:
|
|
202
|
+
first_seen[key] = row
|
|
203
|
+
return errors
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def run_checks(columns, records, rules, problems=None):
|
|
207
|
+
errors = check_structure(problems or [])
|
|
208
|
+
errors += check_schema(columns, rules)
|
|
209
|
+
present = {column: rule for column, rule in rules.items() if column in columns}
|
|
210
|
+
errors += check_missing(records, present)
|
|
211
|
+
errors += check_types_and_ranges(records, present)
|
|
212
|
+
errors += check_formats(records, present)
|
|
213
|
+
errors += check_patterns(records, present)
|
|
214
|
+
errors += check_allowed(records, present)
|
|
215
|
+
errors += check_duplicates(records, present)
|
|
216
|
+
errors += check_duplicate_rows(columns, records)
|
|
217
|
+
order = {column: index for index, column in enumerate(columns)}
|
|
218
|
+
errors.sort(key=lambda e: (e["row"] if e["row"] is not None else 0, order.get(e["column"], 0)))
|
|
219
|
+
return errors
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from . import messages, settings
|
|
4
|
+
from .checks import is_empty
|
|
5
|
+
from .rules import FORMAT_CHECKS, parse_number
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def column_words(name):
|
|
9
|
+
spaced = re.sub(r"([a-z])([A-Z])", r"\1 \2", str(name))
|
|
10
|
+
return {word.lower() for word in re.findall(r"[A-Za-z]+", spaced)}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def share(values, test):
|
|
14
|
+
if len(values) > settings.SAMPLE_SIZE:
|
|
15
|
+
values = values[:: len(values) // settings.SAMPLE_SIZE + 1]
|
|
16
|
+
return sum(1 for value in values if test(value)) / len(values)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def detect_format(name, values):
|
|
20
|
+
words = column_words(name)
|
|
21
|
+
hinted = [fmt for fmt, hints in settings.NAME_HINTS.items() if words & hints]
|
|
22
|
+
for fmt in hinted:
|
|
23
|
+
if share(values, FORMAT_CHECKS[fmt]) >= settings.NAME_MATCH:
|
|
24
|
+
return fmt, None
|
|
25
|
+
if hinted:
|
|
26
|
+
kind = messages.FORMAT_NAMES[hinted[0]]
|
|
27
|
+
return None, messages.NAME_SUGGESTS.format(column=name, kind=kind)
|
|
28
|
+
for fmt in ("email", "date"):
|
|
29
|
+
if share(values, FORMAT_CHECKS[fmt]) >= settings.AGREE:
|
|
30
|
+
return fmt, None
|
|
31
|
+
return None, None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def detect_type(values):
|
|
35
|
+
if share(values, lambda v: parse_number(v, "integer") is not None) >= settings.AGREE:
|
|
36
|
+
return "integer"
|
|
37
|
+
if share(values, lambda v: parse_number(v, "number") is not None) >= settings.AGREE:
|
|
38
|
+
return "number"
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def looks_like_id(column, values):
|
|
43
|
+
if not column_words(column) & settings.ID_WORDS:
|
|
44
|
+
return False
|
|
45
|
+
return len(set(values)) / len(values) >= settings.DISTINCT
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def column_values(column, records):
|
|
49
|
+
return [
|
|
50
|
+
str(record.get(column)).strip()
|
|
51
|
+
for _, record in records
|
|
52
|
+
if not is_empty(record.get(column))
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def detect_column(column, values, total):
|
|
57
|
+
rule = {}
|
|
58
|
+
notes = []
|
|
59
|
+
if len(values) / total >= settings.AGREE:
|
|
60
|
+
rule["required"] = True
|
|
61
|
+
fmt, note = detect_format(column, values)
|
|
62
|
+
if note:
|
|
63
|
+
notes.append(note)
|
|
64
|
+
if fmt:
|
|
65
|
+
rule["format"] = fmt
|
|
66
|
+
else:
|
|
67
|
+
kind = detect_type(values)
|
|
68
|
+
if kind:
|
|
69
|
+
rule["type"] = kind
|
|
70
|
+
if looks_like_id(column, values):
|
|
71
|
+
rule["unique"] = True
|
|
72
|
+
return rule, notes
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def detect_rules(columns, records):
|
|
76
|
+
if not records:
|
|
77
|
+
return {}, [messages.NO_RECORDS]
|
|
78
|
+
if len(records) < settings.MIN_RECORDS:
|
|
79
|
+
note = messages.TOO_FEW_RECORDS.format(found=len(records), needed=settings.MIN_RECORDS)
|
|
80
|
+
return {}, [note]
|
|
81
|
+
rules = {}
|
|
82
|
+
notes = []
|
|
83
|
+
for column in columns:
|
|
84
|
+
values = column_values(column, records)
|
|
85
|
+
if len(values) < settings.MIN_VALUES:
|
|
86
|
+
continue
|
|
87
|
+
rule, column_notes = detect_column(column, values, len(records))
|
|
88
|
+
notes += column_notes
|
|
89
|
+
if rule:
|
|
90
|
+
rules[column] = rule
|
|
91
|
+
return rules, notes
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import json
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from . import messages, settings
|
|
6
|
+
from .errors import DataCheckupError
|
|
7
|
+
from .rules import parse_number
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def safe_cell(value):
|
|
11
|
+
text = "" if value is None else str(value)
|
|
12
|
+
if text.startswith(settings.FORMULA_STARTS):
|
|
13
|
+
return "'" + text
|
|
14
|
+
if text.startswith(settings.SIGN_STARTS) and parse_number(text) is None:
|
|
15
|
+
return "'" + text
|
|
16
|
+
return text
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def check_export_path(path, source):
|
|
20
|
+
if path.resolve() == Path(source).resolve():
|
|
21
|
+
raise DataCheckupError(messages.EXPORT_SAME_FILE)
|
|
22
|
+
if path.suffix.lower() not in settings.EXPORT_SUFFIXES:
|
|
23
|
+
raise DataCheckupError(messages.EXPORT_BAD_TYPE)
|
|
24
|
+
if not path.parent.exists():
|
|
25
|
+
raise DataCheckupError(messages.EXPORT_FOLDER_MISSING.format(folder=path.parent))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def group_by_row(errors):
|
|
29
|
+
problems = {}
|
|
30
|
+
for error in errors:
|
|
31
|
+
if error["row"] is not None:
|
|
32
|
+
problems.setdefault(error["row"], []).append(error)
|
|
33
|
+
return problems
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def export_problem_records(source, columns, records, errors, path):
|
|
37
|
+
path = Path(path)
|
|
38
|
+
check_export_path(path, source)
|
|
39
|
+
problems = group_by_row(errors)
|
|
40
|
+
rows = [(row, record) for row, record in records if row in problems]
|
|
41
|
+
try:
|
|
42
|
+
if path.suffix.lower() == ".csv":
|
|
43
|
+
write_csv(path, columns, rows, problems)
|
|
44
|
+
else:
|
|
45
|
+
write_json(path, columns, rows, problems)
|
|
46
|
+
except OSError as error:
|
|
47
|
+
raise DataCheckupError(messages.EXPORT_WRITE_FAILED.format(path=path, reason=error.strerror)) from None
|
|
48
|
+
return len(rows)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def write_csv(path, columns, rows, problems):
|
|
52
|
+
with open(path, "w", newline="", encoding="utf-8") as f:
|
|
53
|
+
writer = csv.writer(f)
|
|
54
|
+
writer.writerow([safe_cell(c) for c in ["File Row"] + columns + ["Problems"]])
|
|
55
|
+
for row, record in rows:
|
|
56
|
+
summary = "; ".join(f"{e['column']}: {e['reason']}" for e in problems[row])
|
|
57
|
+
cells = [row] + [record.get(c, "") for c in columns] + [summary]
|
|
58
|
+
writer.writerow([safe_cell(cell) for cell in cells])
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def write_json(path, columns, rows, problems):
|
|
62
|
+
output = []
|
|
63
|
+
for row, record in rows:
|
|
64
|
+
output.append({
|
|
65
|
+
"row": row,
|
|
66
|
+
"record": {c: record.get(c) for c in columns},
|
|
67
|
+
"problems": [
|
|
68
|
+
{"column": e["column"], "reason": e["reason"], "suggestion": e["suggestion"]}
|
|
69
|
+
for e in problems[row]
|
|
70
|
+
],
|
|
71
|
+
})
|
|
72
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
73
|
+
json.dump(output, f, indent=2, ensure_ascii=False)
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
EXCEL_HELP = "In Excel, choose File > Save As > CSV UTF-8 (Comma delimited), then check that file."
|
|
2
|
+
|
|
3
|
+
PATH_NOT_TEXT = "The path must be a file name such as 'data.csv'."
|
|
4
|
+
FILE_NOT_FOUND = "File not found: '{path}'. Check the file name and the folder."
|
|
5
|
+
IS_A_FOLDER = "'{path}' is a folder, not a file."
|
|
6
|
+
EXCEL_NOT_SUPPORTED = "Excel files are not supported yet. " + EXCEL_HELP
|
|
7
|
+
UNSUPPORTED_TYPE = "'{suffix}' files are not supported. Use a .csv or .json file."
|
|
8
|
+
NO_EXTENSION = "The file has no extension. Use a .csv or .json file."
|
|
9
|
+
NO_PERMISSION = "No permission to read '{path}'."
|
|
10
|
+
CSV_UNREADABLE = (
|
|
11
|
+
"The CSV file could not be read ({reason}). "
|
|
12
|
+
"Check the file for a quote mark (\") that was opened but never closed."
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
EXCEL_WITH_OTHER_NAME = "This looks like an Excel file saved with another extension. " + EXCEL_HELP
|
|
16
|
+
LOOKS_LIKE_PDF = "This looks like a PDF file, not a CSV or JSON file."
|
|
17
|
+
NOT_A_TEXT_FILE = "This does not look like a text file, so it cannot be read as CSV or JSON."
|
|
18
|
+
ENCODING_UNKNOWN = (
|
|
19
|
+
"The text encoding of this file is not recognised. "
|
|
20
|
+
"Open it in Excel and save it as CSV UTF-8, then try again."
|
|
21
|
+
)
|
|
22
|
+
FILE_EMPTY = "The file is empty."
|
|
23
|
+
|
|
24
|
+
JSON_INVALID = "The file is not valid JSON: {reason} (line {line}, column {column})."
|
|
25
|
+
JSON_IS_OBJECT = (
|
|
26
|
+
"The JSON file contains one object, but DataCheckup needs a list of records "
|
|
27
|
+
"like [{...}, {...}]. If your records are inside a key such as \"data\", "
|
|
28
|
+
"move them to the top level."
|
|
29
|
+
)
|
|
30
|
+
JSON_NOT_LIST = "The JSON file must contain a list of records like [{...}, {...}]."
|
|
31
|
+
JSON_BAD_RECORD = (
|
|
32
|
+
"Record {number} is not a record with named fields. "
|
|
33
|
+
"Every item in the list must look like {{'Name': 'Ravi'}}."
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
COLUMN_WITHOUT_NAME = "Column has no name"
|
|
37
|
+
COLUMN_WITHOUT_NAME_FIX = "Give this column a name in the header row"
|
|
38
|
+
DUPLICATE_COLUMN = "Duplicate column name '{name}'"
|
|
39
|
+
DUPLICATE_COLUMN_FIX = "Give each column a different name"
|
|
40
|
+
EMPTY_ROW = "Row is completely empty"
|
|
41
|
+
EMPTY_ROW_FIX = "Delete the empty row"
|
|
42
|
+
WRONG_VALUE_COUNT = "Row has {found} values but the header has {expected} columns"
|
|
43
|
+
WRONG_VALUE_COUNT_FIX = "Check for a missing or extra separator in this row"
|
|
44
|
+
|
|
45
|
+
RULES_NOT_DICT = "Rules must be a dictionary of column names"
|
|
46
|
+
COLUMN_RULES_NOT_DICT = "Rules for '{column}' must be a dictionary"
|
|
47
|
+
UNKNOWN_RULE = "Unknown rule {names} for column '{column}'"
|
|
48
|
+
UNKNOWN_TYPE = "Unknown type '{value}' for column '{column}'"
|
|
49
|
+
UNKNOWN_FORMAT = "Unknown format '{value}' for column '{column}'"
|
|
50
|
+
LIMIT_NEEDS_NUMBER_TYPE = "'{limit}' needs type 'integer' or 'number' for column '{column}'"
|
|
51
|
+
LIMIT_NOT_A_NUMBER = "'{limit}' must be a number for column '{column}'"
|
|
52
|
+
PATTERN_NOT_TEXT = "The pattern for column '{column}' must be text."
|
|
53
|
+
PATTERN_INVALID = "The pattern for column '{column}' is not a valid pattern: {reason}"
|
|
54
|
+
ALLOWED_NOT_LIST = "'allowed' for column '{column}' must be a list with at least one value, for example ['Male', 'Female']"
|
|
55
|
+
MESSAGE_NOT_TEXT = "'message' for column '{column}' must be text."
|
|
56
|
+
UNKNOWN_COLUMN_RULE = (
|
|
57
|
+
"Rules were given for column '{column}', but the file has no column with that name. "
|
|
58
|
+
"Columns found: {columns}."
|
|
59
|
+
)
|
|
60
|
+
CUSTOM_RULES_APPLIED = "Your own rules were applied to: {columns}. All other columns use the detected rules."
|
|
61
|
+
|
|
62
|
+
PATTERN_MISMATCH = "Value does not match the required pattern {pattern}"
|
|
63
|
+
PATTERN_FIX = "Change the value so it follows the rule set for this column"
|
|
64
|
+
NOT_ALLOWED = "Value is not one of the allowed values"
|
|
65
|
+
NOT_ALLOWED_FIX = "Use one of: {values}"
|
|
66
|
+
|
|
67
|
+
MISSING_COLUMN = "Required column is missing from the file"
|
|
68
|
+
MISSING_COLUMN_FIX = "Add a column named '{column}'"
|
|
69
|
+
MISSING_VALUE = "Missing value"
|
|
70
|
+
MISSING_VALUE_FIX = "Enter a value for {column}"
|
|
71
|
+
WRONG_TYPE = "Invalid data type, expected {expected}"
|
|
72
|
+
WRONG_TYPE_FIX = "Use a number like 30000"
|
|
73
|
+
EXPECTED_NUMBER = "a number"
|
|
74
|
+
EXPECTED_WHOLE_NUMBER = "a whole number"
|
|
75
|
+
BELOW_MINIMUM = "Value is below the minimum of {limit}"
|
|
76
|
+
BELOW_MINIMUM_FIX = "Use a value of {limit} or more"
|
|
77
|
+
ABOVE_MAXIMUM = "Value is above the maximum of {limit}"
|
|
78
|
+
ABOVE_MAXIMUM_FIX = "Use a value of {limit} or less"
|
|
79
|
+
WRONG_FORMAT = "Invalid {format} format"
|
|
80
|
+
DUPLICATE_VALUE = "Duplicate value (first seen in row {row})"
|
|
81
|
+
DUPLICATE_VALUE_FIX = "Make sure each value is unique"
|
|
82
|
+
DUPLICATE_ROW = "Duplicate row (same as row {row})"
|
|
83
|
+
DUPLICATE_ROW_FIX = "Remove the repeated record"
|
|
84
|
+
|
|
85
|
+
FORMAT_FIX = {
|
|
86
|
+
"email": "Use a format like name@example.com",
|
|
87
|
+
"phone": "Use 10 to 13 digits, for example 9876543210",
|
|
88
|
+
"date": "Use a real date like 2026-11-10 or 10-11-2026 (day first)",
|
|
89
|
+
}
|
|
90
|
+
FORMAT_NAMES = {
|
|
91
|
+
"email": "email",
|
|
92
|
+
"phone": "phone number",
|
|
93
|
+
"date": "date",
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
NO_RECORDS = "The file has no records, so there is nothing to check."
|
|
97
|
+
TOO_FEW_RECORDS = (
|
|
98
|
+
"Only {found} records found. At least {needed} are needed "
|
|
99
|
+
"to detect rules, so only repeated rows were checked."
|
|
100
|
+
)
|
|
101
|
+
NAME_SUGGESTS = (
|
|
102
|
+
"Column '{column}' has a name that suggests {kind} values, "
|
|
103
|
+
"but most values do not match, so its format was not checked."
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
EXPORT_SAME_FILE = "The export file must be different from the original file. The original is never changed."
|
|
107
|
+
EXPORT_BAD_TYPE = "The export file name must end with .csv or .json."
|
|
108
|
+
EXPORT_FOLDER_MISSING = "The folder '{folder}' does not exist."
|
|
109
|
+
EXPORT_WRITE_FAILED = "Could not write '{path}': {reason}."
|
|
110
|
+
|
|
111
|
+
ERROR_LABELS = {
|
|
112
|
+
"structure": "File-structure errors",
|
|
113
|
+
"missing": "Missing-value errors",
|
|
114
|
+
"duplicate": "Duplicate errors",
|
|
115
|
+
"duplicate_row": "Duplicate-row errors",
|
|
116
|
+
"invalid_format": "Invalid-format errors",
|
|
117
|
+
"invalid_type": "Data-type errors",
|
|
118
|
+
"out_of_range": "Out-of-range errors",
|
|
119
|
+
"not_allowed": "Not-allowed-value errors",
|
|
120
|
+
"schema": "Schema errors",
|
|
121
|
+
}
|