datacheckup 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datacheckup/__init__.py +25 -0
- datacheckup/checks.py +219 -0
- datacheckup/detector.py +91 -0
- datacheckup/errors.py +2 -0
- datacheckup/exporter.py +73 -0
- datacheckup/messages.py +121 -0
- datacheckup/reader.py +216 -0
- datacheckup/report.py +130 -0
- datacheckup/rules.py +166 -0
- datacheckup/settings.py +63 -0
- datacheckup-0.1.0.dist-info/METADATA +64 -0
- datacheckup-0.1.0.dist-info/RECORD +15 -0
- datacheckup-0.1.0.dist-info/WHEEL +5 -0
- datacheckup-0.1.0.dist-info/licenses/LICENSE +21 -0
- datacheckup-0.1.0.dist-info/top_level.txt +1 -0
datacheckup/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from .checks import run_checks
|
|
2
|
+
from .detector import detect_rules
|
|
3
|
+
from .errors import DataCheckupError
|
|
4
|
+
from .reader import read_file
|
|
5
|
+
from .report import Report
|
|
6
|
+
from .rules import combine_rules, load_rules
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def check(path, rules=None):
|
|
10
|
+
custom = load_rules(rules, partial=True) if rules is not None else {}
|
|
11
|
+
columns, records, problems = read_file(path)
|
|
12
|
+
detected, notes = detect_rules(columns, records)
|
|
13
|
+
final, custom_notes = combine_rules(detected, custom, columns)
|
|
14
|
+
errors = run_checks(columns, records, final, problems)
|
|
15
|
+
return Report(path, columns, records, errors, final, True, notes + custom_notes)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def check_with_rules(path, rules):
|
|
19
|
+
rules = load_rules(rules)
|
|
20
|
+
columns, records, problems = read_file(path)
|
|
21
|
+
errors = run_checks(columns, records, rules, problems)
|
|
22
|
+
return Report(path, columns, records, errors, rules, False, [])
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
__all__ = ["check", "Report", "DataCheckupError"]
|
datacheckup/checks.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from . import messages, settings
|
|
4
|
+
from .rules import FORMAT_CHECKS, parse_number
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def is_empty(value):
|
|
8
|
+
return value is None or str(value).strip() == ""
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def mask_value(value):
|
|
12
|
+
text = str(value).strip()
|
|
13
|
+
if len(text) <= 4:
|
|
14
|
+
return "*" * len(text)
|
|
15
|
+
return text[:2] + "*" * (len(text) - 4) + text[-2:]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def display_value(value, rule):
|
|
19
|
+
if value is None:
|
|
20
|
+
return ""
|
|
21
|
+
text = str(value).strip()
|
|
22
|
+
if rule.get("sensitive") or rule.get("format") in settings.SENSITIVE_FORMATS:
|
|
23
|
+
return mask_value(text)
|
|
24
|
+
return text
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def make_error(row, column, value, error_type, reason, suggestion=""):
|
|
28
|
+
return {
|
|
29
|
+
"row": row,
|
|
30
|
+
"column": column,
|
|
31
|
+
"value": value,
|
|
32
|
+
"error_type": error_type,
|
|
33
|
+
"reason": reason,
|
|
34
|
+
"suggestion": suggestion,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def check_structure(problems):
|
|
39
|
+
return [
|
|
40
|
+
make_error(p["row"], p["column"], "", "structure", p["reason"], p["suggestion"])
|
|
41
|
+
for p in problems
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def check_schema(columns, rules):
|
|
46
|
+
errors = []
|
|
47
|
+
for column, rule in rules.items():
|
|
48
|
+
if column not in columns and rule.get("required"):
|
|
49
|
+
errors.append(make_error(
|
|
50
|
+
None, column, "", "schema",
|
|
51
|
+
messages.MISSING_COLUMN,
|
|
52
|
+
messages.MISSING_COLUMN_FIX.format(column=column),
|
|
53
|
+
))
|
|
54
|
+
return errors
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def check_missing(records, rules):
|
|
58
|
+
errors = []
|
|
59
|
+
for row, record in records:
|
|
60
|
+
for column, rule in rules.items():
|
|
61
|
+
if rule.get("required") and is_empty(record.get(column)):
|
|
62
|
+
errors.append(make_error(
|
|
63
|
+
row, column, "", "missing",
|
|
64
|
+
messages.MISSING_VALUE,
|
|
65
|
+
messages.MISSING_VALUE_FIX.format(column=column),
|
|
66
|
+
))
|
|
67
|
+
return errors
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def check_number(row, column, rule, value):
|
|
71
|
+
kind = rule["type"]
|
|
72
|
+
shown = display_value(value, rule)
|
|
73
|
+
number = parse_number(value, kind)
|
|
74
|
+
if number is None:
|
|
75
|
+
expected = messages.EXPECTED_WHOLE_NUMBER if kind == "integer" else messages.EXPECTED_NUMBER
|
|
76
|
+
return [make_error(
|
|
77
|
+
row, column, shown, "invalid_type",
|
|
78
|
+
messages.WRONG_TYPE.format(expected=expected),
|
|
79
|
+
messages.WRONG_TYPE_FIX,
|
|
80
|
+
)]
|
|
81
|
+
errors = []
|
|
82
|
+
if "min" in rule and number < rule["min"]:
|
|
83
|
+
errors.append(make_error(
|
|
84
|
+
row, column, shown, "out_of_range",
|
|
85
|
+
messages.BELOW_MINIMUM.format(limit=rule["min"]),
|
|
86
|
+
messages.BELOW_MINIMUM_FIX.format(limit=rule["min"]),
|
|
87
|
+
))
|
|
88
|
+
if "max" in rule and number > rule["max"]:
|
|
89
|
+
errors.append(make_error(
|
|
90
|
+
row, column, shown, "out_of_range",
|
|
91
|
+
messages.ABOVE_MAXIMUM.format(limit=rule["max"]),
|
|
92
|
+
messages.ABOVE_MAXIMUM_FIX.format(limit=rule["max"]),
|
|
93
|
+
))
|
|
94
|
+
return errors
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def check_types_and_ranges(records, rules):
|
|
98
|
+
errors = []
|
|
99
|
+
for row, record in records:
|
|
100
|
+
for column, rule in rules.items():
|
|
101
|
+
if rule.get("type") not in settings.NUMBER_TYPES:
|
|
102
|
+
continue
|
|
103
|
+
value = record.get(column)
|
|
104
|
+
if not is_empty(value):
|
|
105
|
+
errors += check_number(row, column, rule, value)
|
|
106
|
+
return errors
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def check_formats(records, rules):
|
|
110
|
+
errors = []
|
|
111
|
+
for row, record in records:
|
|
112
|
+
for column, rule in rules.items():
|
|
113
|
+
name = rule.get("format")
|
|
114
|
+
value = record.get(column)
|
|
115
|
+
if not name or is_empty(value):
|
|
116
|
+
continue
|
|
117
|
+
if not FORMAT_CHECKS[name](str(value).strip()):
|
|
118
|
+
errors.append(make_error(
|
|
119
|
+
row, column, display_value(value, rule), "invalid_format",
|
|
120
|
+
messages.WRONG_FORMAT.format(format=name),
|
|
121
|
+
messages.FORMAT_FIX[name],
|
|
122
|
+
))
|
|
123
|
+
return errors
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def check_patterns(records, rules):
|
|
127
|
+
errors = []
|
|
128
|
+
for column, rule in rules.items():
|
|
129
|
+
if "pattern" not in rule:
|
|
130
|
+
continue
|
|
131
|
+
pattern = re.compile(rule["pattern"])
|
|
132
|
+
reason = rule.get("message") or messages.PATTERN_MISMATCH.format(pattern=rule["pattern"])
|
|
133
|
+
for row, record in records:
|
|
134
|
+
value = record.get(column)
|
|
135
|
+
if is_empty(value) or pattern.fullmatch(str(value).strip()):
|
|
136
|
+
continue
|
|
137
|
+
errors.append(make_error(
|
|
138
|
+
row, column, display_value(value, rule), "invalid_format",
|
|
139
|
+
reason, messages.PATTERN_FIX,
|
|
140
|
+
))
|
|
141
|
+
return errors
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def check_allowed(records, rules):
|
|
145
|
+
errors = []
|
|
146
|
+
for column, rule in rules.items():
|
|
147
|
+
if "allowed" not in rule:
|
|
148
|
+
continue
|
|
149
|
+
allowed = [str(item).strip() for item in rule["allowed"]]
|
|
150
|
+
reason = rule.get("message") or messages.NOT_ALLOWED
|
|
151
|
+
fix = messages.NOT_ALLOWED_FIX.format(values=", ".join(allowed))
|
|
152
|
+
for row, record in records:
|
|
153
|
+
value = record.get(column)
|
|
154
|
+
if is_empty(value) or str(value).strip() in allowed:
|
|
155
|
+
continue
|
|
156
|
+
errors.append(make_error(
|
|
157
|
+
row, column, display_value(value, rule), "not_allowed", reason, fix,
|
|
158
|
+
))
|
|
159
|
+
return errors
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def check_duplicates(records, rules):
|
|
163
|
+
errors = []
|
|
164
|
+
for column, rule in rules.items():
|
|
165
|
+
if not rule.get("unique"):
|
|
166
|
+
continue
|
|
167
|
+
first_seen = {}
|
|
168
|
+
for row, record in records:
|
|
169
|
+
value = record.get(column)
|
|
170
|
+
if is_empty(value):
|
|
171
|
+
continue
|
|
172
|
+
key = str(value).strip()
|
|
173
|
+
if key in first_seen:
|
|
174
|
+
errors.append(make_error(
|
|
175
|
+
row, column, display_value(value, rule), "duplicate",
|
|
176
|
+
messages.DUPLICATE_VALUE.format(row=first_seen[key]),
|
|
177
|
+
messages.DUPLICATE_VALUE_FIX,
|
|
178
|
+
))
|
|
179
|
+
else:
|
|
180
|
+
first_seen[key] = row
|
|
181
|
+
return errors
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def row_key(columns, record):
|
|
185
|
+
return tuple("" if is_empty(record.get(c)) else str(record.get(c)).strip() for c in columns)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def check_duplicate_rows(columns, records):
|
|
189
|
+
errors = []
|
|
190
|
+
first_seen = {}
|
|
191
|
+
for row, record in records:
|
|
192
|
+
key = row_key(columns, record)
|
|
193
|
+
if not any(key):
|
|
194
|
+
continue
|
|
195
|
+
if key in first_seen:
|
|
196
|
+
errors.append(make_error(
|
|
197
|
+
row, settings.ENTIRE_ROW, "", "duplicate_row",
|
|
198
|
+
messages.DUPLICATE_ROW.format(row=first_seen[key]),
|
|
199
|
+
messages.DUPLICATE_ROW_FIX,
|
|
200
|
+
))
|
|
201
|
+
else:
|
|
202
|
+
first_seen[key] = row
|
|
203
|
+
return errors
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def run_checks(columns, records, rules, problems=None):
|
|
207
|
+
errors = check_structure(problems or [])
|
|
208
|
+
errors += check_schema(columns, rules)
|
|
209
|
+
present = {column: rule for column, rule in rules.items() if column in columns}
|
|
210
|
+
errors += check_missing(records, present)
|
|
211
|
+
errors += check_types_and_ranges(records, present)
|
|
212
|
+
errors += check_formats(records, present)
|
|
213
|
+
errors += check_patterns(records, present)
|
|
214
|
+
errors += check_allowed(records, present)
|
|
215
|
+
errors += check_duplicates(records, present)
|
|
216
|
+
errors += check_duplicate_rows(columns, records)
|
|
217
|
+
order = {column: index for index, column in enumerate(columns)}
|
|
218
|
+
errors.sort(key=lambda e: (e["row"] if e["row"] is not None else 0, order.get(e["column"], 0)))
|
|
219
|
+
return errors
|
datacheckup/detector.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from . import messages, settings
|
|
4
|
+
from .checks import is_empty
|
|
5
|
+
from .rules import FORMAT_CHECKS, parse_number
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def column_words(name):
|
|
9
|
+
spaced = re.sub(r"([a-z])([A-Z])", r"\1 \2", str(name))
|
|
10
|
+
return {word.lower() for word in re.findall(r"[A-Za-z]+", spaced)}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def share(values, test):
|
|
14
|
+
if len(values) > settings.SAMPLE_SIZE:
|
|
15
|
+
values = values[:: len(values) // settings.SAMPLE_SIZE + 1]
|
|
16
|
+
return sum(1 for value in values if test(value)) / len(values)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def detect_format(name, values):
|
|
20
|
+
words = column_words(name)
|
|
21
|
+
hinted = [fmt for fmt, hints in settings.NAME_HINTS.items() if words & hints]
|
|
22
|
+
for fmt in hinted:
|
|
23
|
+
if share(values, FORMAT_CHECKS[fmt]) >= settings.NAME_MATCH:
|
|
24
|
+
return fmt, None
|
|
25
|
+
if hinted:
|
|
26
|
+
kind = messages.FORMAT_NAMES[hinted[0]]
|
|
27
|
+
return None, messages.NAME_SUGGESTS.format(column=name, kind=kind)
|
|
28
|
+
for fmt in ("email", "date"):
|
|
29
|
+
if share(values, FORMAT_CHECKS[fmt]) >= settings.AGREE:
|
|
30
|
+
return fmt, None
|
|
31
|
+
return None, None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def detect_type(values):
|
|
35
|
+
if share(values, lambda v: parse_number(v, "integer") is not None) >= settings.AGREE:
|
|
36
|
+
return "integer"
|
|
37
|
+
if share(values, lambda v: parse_number(v, "number") is not None) >= settings.AGREE:
|
|
38
|
+
return "number"
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def looks_like_id(column, values):
|
|
43
|
+
if not column_words(column) & settings.ID_WORDS:
|
|
44
|
+
return False
|
|
45
|
+
return len(set(values)) / len(values) >= settings.DISTINCT
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def column_values(column, records):
|
|
49
|
+
return [
|
|
50
|
+
str(record.get(column)).strip()
|
|
51
|
+
for _, record in records
|
|
52
|
+
if not is_empty(record.get(column))
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def detect_column(column, values, total):
|
|
57
|
+
rule = {}
|
|
58
|
+
notes = []
|
|
59
|
+
if len(values) / total >= settings.AGREE:
|
|
60
|
+
rule["required"] = True
|
|
61
|
+
fmt, note = detect_format(column, values)
|
|
62
|
+
if note:
|
|
63
|
+
notes.append(note)
|
|
64
|
+
if fmt:
|
|
65
|
+
rule["format"] = fmt
|
|
66
|
+
else:
|
|
67
|
+
kind = detect_type(values)
|
|
68
|
+
if kind:
|
|
69
|
+
rule["type"] = kind
|
|
70
|
+
if looks_like_id(column, values):
|
|
71
|
+
rule["unique"] = True
|
|
72
|
+
return rule, notes
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def detect_rules(columns, records):
|
|
76
|
+
if not records:
|
|
77
|
+
return {}, [messages.NO_RECORDS]
|
|
78
|
+
if len(records) < settings.MIN_RECORDS:
|
|
79
|
+
note = messages.TOO_FEW_RECORDS.format(found=len(records), needed=settings.MIN_RECORDS)
|
|
80
|
+
return {}, [note]
|
|
81
|
+
rules = {}
|
|
82
|
+
notes = []
|
|
83
|
+
for column in columns:
|
|
84
|
+
values = column_values(column, records)
|
|
85
|
+
if len(values) < settings.MIN_VALUES:
|
|
86
|
+
continue
|
|
87
|
+
rule, column_notes = detect_column(column, values, len(records))
|
|
88
|
+
notes += column_notes
|
|
89
|
+
if rule:
|
|
90
|
+
rules[column] = rule
|
|
91
|
+
return rules, notes
|
datacheckup/errors.py
ADDED
datacheckup/exporter.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import json
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from . import messages, settings
|
|
6
|
+
from .errors import DataCheckupError
|
|
7
|
+
from .rules import parse_number
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def safe_cell(value):
|
|
11
|
+
text = "" if value is None else str(value)
|
|
12
|
+
if text.startswith(settings.FORMULA_STARTS):
|
|
13
|
+
return "'" + text
|
|
14
|
+
if text.startswith(settings.SIGN_STARTS) and parse_number(text) is None:
|
|
15
|
+
return "'" + text
|
|
16
|
+
return text
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def check_export_path(path, source):
|
|
20
|
+
if path.resolve() == Path(source).resolve():
|
|
21
|
+
raise DataCheckupError(messages.EXPORT_SAME_FILE)
|
|
22
|
+
if path.suffix.lower() not in settings.EXPORT_SUFFIXES:
|
|
23
|
+
raise DataCheckupError(messages.EXPORT_BAD_TYPE)
|
|
24
|
+
if not path.parent.exists():
|
|
25
|
+
raise DataCheckupError(messages.EXPORT_FOLDER_MISSING.format(folder=path.parent))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def group_by_row(errors):
|
|
29
|
+
problems = {}
|
|
30
|
+
for error in errors:
|
|
31
|
+
if error["row"] is not None:
|
|
32
|
+
problems.setdefault(error["row"], []).append(error)
|
|
33
|
+
return problems
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def export_problem_records(source, columns, records, errors, path):
|
|
37
|
+
path = Path(path)
|
|
38
|
+
check_export_path(path, source)
|
|
39
|
+
problems = group_by_row(errors)
|
|
40
|
+
rows = [(row, record) for row, record in records if row in problems]
|
|
41
|
+
try:
|
|
42
|
+
if path.suffix.lower() == ".csv":
|
|
43
|
+
write_csv(path, columns, rows, problems)
|
|
44
|
+
else:
|
|
45
|
+
write_json(path, columns, rows, problems)
|
|
46
|
+
except OSError as error:
|
|
47
|
+
raise DataCheckupError(messages.EXPORT_WRITE_FAILED.format(path=path, reason=error.strerror)) from None
|
|
48
|
+
return len(rows)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def write_csv(path, columns, rows, problems):
|
|
52
|
+
with open(path, "w", newline="", encoding="utf-8") as f:
|
|
53
|
+
writer = csv.writer(f)
|
|
54
|
+
writer.writerow([safe_cell(c) for c in ["File Row"] + columns + ["Problems"]])
|
|
55
|
+
for row, record in rows:
|
|
56
|
+
summary = "; ".join(f"{e['column']}: {e['reason']}" for e in problems[row])
|
|
57
|
+
cells = [row] + [record.get(c, "") for c in columns] + [summary]
|
|
58
|
+
writer.writerow([safe_cell(cell) for cell in cells])
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def write_json(path, columns, rows, problems):
|
|
62
|
+
output = []
|
|
63
|
+
for row, record in rows:
|
|
64
|
+
output.append({
|
|
65
|
+
"row": row,
|
|
66
|
+
"record": {c: record.get(c) for c in columns},
|
|
67
|
+
"problems": [
|
|
68
|
+
{"column": e["column"], "reason": e["reason"], "suggestion": e["suggestion"]}
|
|
69
|
+
for e in problems[row]
|
|
70
|
+
],
|
|
71
|
+
})
|
|
72
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
73
|
+
json.dump(output, f, indent=2, ensure_ascii=False)
|
datacheckup/messages.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
EXCEL_HELP = "In Excel, choose File > Save As > CSV UTF-8 (Comma delimited), then check that file."
|
|
2
|
+
|
|
3
|
+
PATH_NOT_TEXT = "The path must be a file name such as 'data.csv'."
|
|
4
|
+
FILE_NOT_FOUND = "File not found: '{path}'. Check the file name and the folder."
|
|
5
|
+
IS_A_FOLDER = "'{path}' is a folder, not a file."
|
|
6
|
+
EXCEL_NOT_SUPPORTED = "Excel files are not supported yet. " + EXCEL_HELP
|
|
7
|
+
UNSUPPORTED_TYPE = "'{suffix}' files are not supported. Use a .csv or .json file."
|
|
8
|
+
NO_EXTENSION = "The file has no extension. Use a .csv or .json file."
|
|
9
|
+
NO_PERMISSION = "No permission to read '{path}'."
|
|
10
|
+
CSV_UNREADABLE = (
|
|
11
|
+
"The CSV file could not be read ({reason}). "
|
|
12
|
+
"Check the file for a quote mark (\") that was opened but never closed."
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
EXCEL_WITH_OTHER_NAME = "This looks like an Excel file saved with another extension. " + EXCEL_HELP
|
|
16
|
+
LOOKS_LIKE_PDF = "This looks like a PDF file, not a CSV or JSON file."
|
|
17
|
+
NOT_A_TEXT_FILE = "This does not look like a text file, so it cannot be read as CSV or JSON."
|
|
18
|
+
ENCODING_UNKNOWN = (
|
|
19
|
+
"The text encoding of this file is not recognised. "
|
|
20
|
+
"Open it in Excel and save it as CSV UTF-8, then try again."
|
|
21
|
+
)
|
|
22
|
+
FILE_EMPTY = "The file is empty."
|
|
23
|
+
|
|
24
|
+
JSON_INVALID = "The file is not valid JSON: {reason} (line {line}, column {column})."
|
|
25
|
+
JSON_IS_OBJECT = (
|
|
26
|
+
"The JSON file contains one object, but DataCheckup needs a list of records "
|
|
27
|
+
"like [{...}, {...}]. If your records are inside a key such as \"data\", "
|
|
28
|
+
"move them to the top level."
|
|
29
|
+
)
|
|
30
|
+
JSON_NOT_LIST = "The JSON file must contain a list of records like [{...}, {...}]."
|
|
31
|
+
JSON_BAD_RECORD = (
|
|
32
|
+
"Record {number} is not a record with named fields. "
|
|
33
|
+
"Every item in the list must look like {{'Name': 'Ravi'}}."
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
COLUMN_WITHOUT_NAME = "Column has no name"
|
|
37
|
+
COLUMN_WITHOUT_NAME_FIX = "Give this column a name in the header row"
|
|
38
|
+
DUPLICATE_COLUMN = "Duplicate column name '{name}'"
|
|
39
|
+
DUPLICATE_COLUMN_FIX = "Give each column a different name"
|
|
40
|
+
EMPTY_ROW = "Row is completely empty"
|
|
41
|
+
EMPTY_ROW_FIX = "Delete the empty row"
|
|
42
|
+
WRONG_VALUE_COUNT = "Row has {found} values but the header has {expected} columns"
|
|
43
|
+
WRONG_VALUE_COUNT_FIX = "Check for a missing or extra separator in this row"
|
|
44
|
+
|
|
45
|
+
RULES_NOT_DICT = "Rules must be a dictionary of column names"
|
|
46
|
+
COLUMN_RULES_NOT_DICT = "Rules for '{column}' must be a dictionary"
|
|
47
|
+
UNKNOWN_RULE = "Unknown rule {names} for column '{column}'"
|
|
48
|
+
UNKNOWN_TYPE = "Unknown type '{value}' for column '{column}'"
|
|
49
|
+
UNKNOWN_FORMAT = "Unknown format '{value}' for column '{column}'"
|
|
50
|
+
LIMIT_NEEDS_NUMBER_TYPE = "'{limit}' needs type 'integer' or 'number' for column '{column}'"
|
|
51
|
+
LIMIT_NOT_A_NUMBER = "'{limit}' must be a number for column '{column}'"
|
|
52
|
+
PATTERN_NOT_TEXT = "The pattern for column '{column}' must be text."
|
|
53
|
+
PATTERN_INVALID = "The pattern for column '{column}' is not a valid pattern: {reason}"
|
|
54
|
+
ALLOWED_NOT_LIST = "'allowed' for column '{column}' must be a list with at least one value, for example ['Male', 'Female']"
|
|
55
|
+
MESSAGE_NOT_TEXT = "'message' for column '{column}' must be text."
|
|
56
|
+
UNKNOWN_COLUMN_RULE = (
|
|
57
|
+
"Rules were given for column '{column}', but the file has no column with that name. "
|
|
58
|
+
"Columns found: {columns}."
|
|
59
|
+
)
|
|
60
|
+
CUSTOM_RULES_APPLIED = "Your own rules were applied to: {columns}. All other columns use the detected rules."
|
|
61
|
+
|
|
62
|
+
PATTERN_MISMATCH = "Value does not match the required pattern {pattern}"
|
|
63
|
+
PATTERN_FIX = "Change the value so it follows the rule set for this column"
|
|
64
|
+
NOT_ALLOWED = "Value is not one of the allowed values"
|
|
65
|
+
NOT_ALLOWED_FIX = "Use one of: {values}"
|
|
66
|
+
|
|
67
|
+
MISSING_COLUMN = "Required column is missing from the file"
|
|
68
|
+
MISSING_COLUMN_FIX = "Add a column named '{column}'"
|
|
69
|
+
MISSING_VALUE = "Missing value"
|
|
70
|
+
MISSING_VALUE_FIX = "Enter a value for {column}"
|
|
71
|
+
WRONG_TYPE = "Invalid data type, expected {expected}"
|
|
72
|
+
WRONG_TYPE_FIX = "Use a number like 30000"
|
|
73
|
+
EXPECTED_NUMBER = "a number"
|
|
74
|
+
EXPECTED_WHOLE_NUMBER = "a whole number"
|
|
75
|
+
BELOW_MINIMUM = "Value is below the minimum of {limit}"
|
|
76
|
+
BELOW_MINIMUM_FIX = "Use a value of {limit} or more"
|
|
77
|
+
ABOVE_MAXIMUM = "Value is above the maximum of {limit}"
|
|
78
|
+
ABOVE_MAXIMUM_FIX = "Use a value of {limit} or less"
|
|
79
|
+
WRONG_FORMAT = "Invalid {format} format"
|
|
80
|
+
DUPLICATE_VALUE = "Duplicate value (first seen in row {row})"
|
|
81
|
+
DUPLICATE_VALUE_FIX = "Make sure each value is unique"
|
|
82
|
+
DUPLICATE_ROW = "Duplicate row (same as row {row})"
|
|
83
|
+
DUPLICATE_ROW_FIX = "Remove the repeated record"
|
|
84
|
+
|
|
85
|
+
FORMAT_FIX = {
|
|
86
|
+
"email": "Use a format like name@example.com",
|
|
87
|
+
"phone": "Use 10 to 13 digits, for example 9876543210",
|
|
88
|
+
"date": "Use a real date like 2026-11-10 or 10-11-2026 (day first)",
|
|
89
|
+
}
|
|
90
|
+
FORMAT_NAMES = {
|
|
91
|
+
"email": "email",
|
|
92
|
+
"phone": "phone number",
|
|
93
|
+
"date": "date",
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
NO_RECORDS = "The file has no records, so there is nothing to check."
|
|
97
|
+
TOO_FEW_RECORDS = (
|
|
98
|
+
"Only {found} records found. At least {needed} are needed "
|
|
99
|
+
"to detect rules, so only repeated rows were checked."
|
|
100
|
+
)
|
|
101
|
+
NAME_SUGGESTS = (
|
|
102
|
+
"Column '{column}' has a name that suggests {kind} values, "
|
|
103
|
+
"but most values do not match, so its format was not checked."
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
EXPORT_SAME_FILE = "The export file must be different from the original file. The original is never changed."
|
|
107
|
+
EXPORT_BAD_TYPE = "The export file name must end with .csv or .json."
|
|
108
|
+
EXPORT_FOLDER_MISSING = "The folder '{folder}' does not exist."
|
|
109
|
+
EXPORT_WRITE_FAILED = "Could not write '{path}': {reason}."
|
|
110
|
+
|
|
111
|
+
ERROR_LABELS = {
|
|
112
|
+
"structure": "File-structure errors",
|
|
113
|
+
"missing": "Missing-value errors",
|
|
114
|
+
"duplicate": "Duplicate errors",
|
|
115
|
+
"duplicate_row": "Duplicate-row errors",
|
|
116
|
+
"invalid_format": "Invalid-format errors",
|
|
117
|
+
"invalid_type": "Data-type errors",
|
|
118
|
+
"out_of_range": "Out-of-range errors",
|
|
119
|
+
"not_allowed": "Not-allowed-value errors",
|
|
120
|
+
"schema": "Schema errors",
|
|
121
|
+
}
|
datacheckup/reader.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from . import messages, settings
|
|
7
|
+
from .errors import DataCheckupError
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def read_file(path):
|
|
11
|
+
path = check_path(path)
|
|
12
|
+
try:
|
|
13
|
+
if path.suffix.lower() == ".json":
|
|
14
|
+
return read_json(path)
|
|
15
|
+
return read_csv(path)
|
|
16
|
+
except PermissionError:
|
|
17
|
+
raise DataCheckupError(messages.NO_PERMISSION.format(path=path)) from None
|
|
18
|
+
except csv.Error as error:
|
|
19
|
+
raise DataCheckupError(messages.CSV_UNREADABLE.format(reason=error)) from None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def check_path(path):
|
|
23
|
+
if not isinstance(path, (str, os.PathLike)):
|
|
24
|
+
raise DataCheckupError(messages.PATH_NOT_TEXT)
|
|
25
|
+
path = Path(path)
|
|
26
|
+
if not path.exists():
|
|
27
|
+
raise DataCheckupError(messages.FILE_NOT_FOUND.format(path=path))
|
|
28
|
+
if path.is_dir():
|
|
29
|
+
raise DataCheckupError(messages.IS_A_FOLDER.format(path=path))
|
|
30
|
+
check_file_type(path.suffix.lower())
|
|
31
|
+
return path
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def check_file_type(suffix):
|
|
35
|
+
if suffix in settings.EXCEL_SUFFIXES:
|
|
36
|
+
raise DataCheckupError(messages.EXCEL_NOT_SUPPORTED)
|
|
37
|
+
if suffix in settings.SUPPORTED_SUFFIXES:
|
|
38
|
+
return
|
|
39
|
+
if suffix:
|
|
40
|
+
raise DataCheckupError(messages.UNSUPPORTED_TYPE.format(suffix=suffix))
|
|
41
|
+
raise DataCheckupError(messages.NO_EXTENSION)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def read_head(path):
|
|
45
|
+
with open(path, "rb") as f:
|
|
46
|
+
return f.read(settings.HEAD_BYTES)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def reject_other_file_types(head):
|
|
50
|
+
if head.startswith(settings.EXCEL_SIGNATURE):
|
|
51
|
+
raise DataCheckupError(messages.EXCEL_WITH_OTHER_NAME)
|
|
52
|
+
if head.startswith(settings.PDF_SIGNATURE):
|
|
53
|
+
raise DataCheckupError(messages.LOOKS_LIKE_PDF)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def reject_binary(head):
|
|
57
|
+
odd = sum(1 for byte in head if byte < 32 and byte not in settings.TEXT_CONTROL_BYTES)
|
|
58
|
+
if head and (b"\x00" in head or odd / len(head) > settings.BINARY_LIMIT):
|
|
59
|
+
raise DataCheckupError(messages.NOT_A_TEXT_FILE)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def can_read(path, encoding):
|
|
63
|
+
try:
|
|
64
|
+
with open(path, encoding=encoding) as f:
|
|
65
|
+
while f.read(settings.READ_CHUNK):
|
|
66
|
+
pass
|
|
67
|
+
return True
|
|
68
|
+
except UnicodeDecodeError:
|
|
69
|
+
return False
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def find_encoding(path):
|
|
73
|
+
head = read_head(path)
|
|
74
|
+
reject_other_file_types(head)
|
|
75
|
+
if head.startswith(settings.UTF16_MARKS):
|
|
76
|
+
return "utf-16"
|
|
77
|
+
reject_binary(head)
|
|
78
|
+
for encoding in settings.ENCODINGS:
|
|
79
|
+
if can_read(path, encoding):
|
|
80
|
+
return encoding
|
|
81
|
+
raise DataCheckupError(messages.ENCODING_UNKNOWN)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def first_line(f):
|
|
85
|
+
for _ in range(settings.HEADER_SEARCH_LINES):
|
|
86
|
+
line = f.readline()
|
|
87
|
+
if not line:
|
|
88
|
+
return ""
|
|
89
|
+
if line.strip():
|
|
90
|
+
return line
|
|
91
|
+
return ""
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def find_delimiter(f):
|
|
95
|
+
line = first_line(f)
|
|
96
|
+
f.seek(0)
|
|
97
|
+
best, best_count = ",", 1
|
|
98
|
+
for candidate in settings.DELIMITERS:
|
|
99
|
+
count = len(next(csv.reader([line], delimiter=candidate), []))
|
|
100
|
+
if count > best_count:
|
|
101
|
+
best, best_count = candidate, count
|
|
102
|
+
return best
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def make_problem(row, column, reason, suggestion):
|
|
106
|
+
return {"row": row, "column": column, "reason": reason, "suggestion": suggestion}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def unique_name(name, seen):
|
|
110
|
+
count = 2
|
|
111
|
+
while f"{name} ({count})" in seen:
|
|
112
|
+
count += 1
|
|
113
|
+
return f"{name} ({count})"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def clean_header(header, row):
|
|
117
|
+
columns = []
|
|
118
|
+
problems = []
|
|
119
|
+
seen = set()
|
|
120
|
+
for index, raw in enumerate(header, start=1):
|
|
121
|
+
name = raw.strip()
|
|
122
|
+
if not name:
|
|
123
|
+
name = f"Column {index}"
|
|
124
|
+
problems.append(make_problem(
|
|
125
|
+
row, name, messages.COLUMN_WITHOUT_NAME, messages.COLUMN_WITHOUT_NAME_FIX,
|
|
126
|
+
))
|
|
127
|
+
elif name in seen:
|
|
128
|
+
problems.append(make_problem(
|
|
129
|
+
row, name, messages.DUPLICATE_COLUMN.format(name=name), messages.DUPLICATE_COLUMN_FIX,
|
|
130
|
+
))
|
|
131
|
+
name = unique_name(name, seen)
|
|
132
|
+
seen.add(name)
|
|
133
|
+
columns.append(name)
|
|
134
|
+
return columns, problems
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def find_header(rows):
|
|
138
|
+
for number, values in rows:
|
|
139
|
+
if any(value.strip() for value in values):
|
|
140
|
+
return number, values
|
|
141
|
+
raise DataCheckupError(messages.FILE_EMPTY)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def read_row(number, values, columns):
|
|
145
|
+
if not any(value.strip() for value in values):
|
|
146
|
+
problem = make_problem(number, settings.ENTIRE_ROW, messages.EMPTY_ROW, messages.EMPTY_ROW_FIX)
|
|
147
|
+
return None, problem
|
|
148
|
+
problem = None
|
|
149
|
+
if len(values) != len(columns):
|
|
150
|
+
problem = make_problem(
|
|
151
|
+
number, settings.ENTIRE_ROW,
|
|
152
|
+
messages.WRONG_VALUE_COUNT.format(found=len(values), expected=len(columns)),
|
|
153
|
+
messages.WRONG_VALUE_COUNT_FIX,
|
|
154
|
+
)
|
|
155
|
+
padded = values + [""] * (len(columns) - len(values))
|
|
156
|
+
return dict(zip(columns, padded)), problem
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def read_csv(path):
|
|
160
|
+
encoding = find_encoding(path)
|
|
161
|
+
with open(path, newline="", encoding=encoding) as f:
|
|
162
|
+
delimiter = find_delimiter(f)
|
|
163
|
+
rows = enumerate(csv.reader(f, delimiter=delimiter), start=1)
|
|
164
|
+
header_row, header = find_header(rows)
|
|
165
|
+
columns, problems = clean_header(header, header_row)
|
|
166
|
+
records = []
|
|
167
|
+
for number, values in rows:
|
|
168
|
+
record, problem = read_row(number, values, columns)
|
|
169
|
+
if problem:
|
|
170
|
+
problems.append(problem)
|
|
171
|
+
if record is not None:
|
|
172
|
+
records.append((number, record))
|
|
173
|
+
return columns, records, problems
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def read_text(path):
|
|
177
|
+
encoding = find_encoding(path)
|
|
178
|
+
with open(path, encoding=encoding) as f:
|
|
179
|
+
return f.read()
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def parse_json(text):
|
|
183
|
+
try:
|
|
184
|
+
return json.loads(text)
|
|
185
|
+
except json.JSONDecodeError as error:
|
|
186
|
+
raise DataCheckupError(messages.JSON_INVALID.format(
|
|
187
|
+
reason=error.msg, line=error.lineno, column=error.colno,
|
|
188
|
+
)) from None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def check_json_shape(data):
|
|
192
|
+
if isinstance(data, dict):
|
|
193
|
+
raise DataCheckupError(messages.JSON_IS_OBJECT)
|
|
194
|
+
if not isinstance(data, list):
|
|
195
|
+
raise DataCheckupError(messages.JSON_NOT_LIST)
|
|
196
|
+
for number, item in enumerate(data, start=1):
|
|
197
|
+
if not isinstance(item, dict):
|
|
198
|
+
raise DataCheckupError(messages.JSON_BAD_RECORD.format(number=number))
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def find_json_columns(data):
|
|
202
|
+
columns = []
|
|
203
|
+
for item in data:
|
|
204
|
+
for key in item:
|
|
205
|
+
if key not in columns:
|
|
206
|
+
columns.append(key)
|
|
207
|
+
return columns
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def read_json(path):
|
|
211
|
+
text = read_text(path)
|
|
212
|
+
if not text.strip():
|
|
213
|
+
raise DataCheckupError(messages.FILE_EMPTY)
|
|
214
|
+
data = parse_json(text)
|
|
215
|
+
check_json_shape(data)
|
|
216
|
+
return find_json_columns(data), list(enumerate(data, start=1)), []
|
datacheckup/report.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
from . import messages, settings
|
|
2
|
+
from .exporter import export_problem_records
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def tidy(text, limit):
|
|
6
|
+
text = " ".join(str(text).split())
|
|
7
|
+
if len(text) <= limit:
|
|
8
|
+
return text
|
|
9
|
+
return text[: limit - 3] + "..."
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def describe_rule(rule):
|
|
13
|
+
parts = []
|
|
14
|
+
if rule.get("required"):
|
|
15
|
+
parts.append("required")
|
|
16
|
+
if rule.get("unique"):
|
|
17
|
+
parts.append("unique")
|
|
18
|
+
kind = rule.get("type")
|
|
19
|
+
if kind == "integer":
|
|
20
|
+
parts.append("whole number")
|
|
21
|
+
elif kind == "number":
|
|
22
|
+
parts.append("number")
|
|
23
|
+
if rule.get("format"):
|
|
24
|
+
parts.append(f"{rule['format']} format")
|
|
25
|
+
if "min" in rule:
|
|
26
|
+
parts.append(f"min {rule['min']}")
|
|
27
|
+
if "max" in rule:
|
|
28
|
+
parts.append(f"max {rule['max']}")
|
|
29
|
+
if rule.get("pattern"):
|
|
30
|
+
parts.append("custom pattern")
|
|
31
|
+
if rule.get("allowed"):
|
|
32
|
+
parts.append("allowed values only")
|
|
33
|
+
return ", ".join(parts)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Report:
|
|
37
|
+
def __init__(self, source, columns, records, errors, rules=None, detected=False, notes=None):
|
|
38
|
+
self.source = str(source)
|
|
39
|
+
self.columns = columns
|
|
40
|
+
self.records = records
|
|
41
|
+
self.errors = errors
|
|
42
|
+
self.rules = rules or {}
|
|
43
|
+
self.detected = detected
|
|
44
|
+
self.notes = notes or []
|
|
45
|
+
self.summary = self.build_summary()
|
|
46
|
+
|
|
47
|
+
def build_summary(self):
|
|
48
|
+
total = len(self.records)
|
|
49
|
+
record_rows = {row for row, _ in self.records}
|
|
50
|
+
bad_rows = {e["row"] for e in self.errors if e["row"] in record_rows}
|
|
51
|
+
by_type = {}
|
|
52
|
+
for error in self.errors:
|
|
53
|
+
by_type[error["error_type"]] = by_type.get(error["error_type"], 0) + 1
|
|
54
|
+
return {
|
|
55
|
+
"total_records": total,
|
|
56
|
+
"records_with_issues": len(bad_rows),
|
|
57
|
+
"records_without_issues": total - len(bad_rows),
|
|
58
|
+
"total_errors": len(self.errors),
|
|
59
|
+
"errors_by_type": by_type,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
def has_errors(self):
|
|
63
|
+
return len(self.errors) > 0
|
|
64
|
+
|
|
65
|
+
def summary_lines(self):
|
|
66
|
+
s = self.summary
|
|
67
|
+
lines = [
|
|
68
|
+
"Dataset Summary",
|
|
69
|
+
f" Total records: {s['total_records']}",
|
|
70
|
+
f" Records with issues: {s['records_with_issues']}",
|
|
71
|
+
f" Records without issues: {s['records_without_issues']}",
|
|
72
|
+
f" Total errors: {s['total_errors']}",
|
|
73
|
+
]
|
|
74
|
+
for key, label in messages.ERROR_LABELS.items():
|
|
75
|
+
if s["errors_by_type"].get(key):
|
|
76
|
+
lines.append(f" {label}: {s['errors_by_type'][key]}")
|
|
77
|
+
return lines
|
|
78
|
+
|
|
79
|
+
def rules_lines(self):
|
|
80
|
+
title = "Rules used (detected automatically)" if self.detected else "Rules used"
|
|
81
|
+
if not self.rules:
|
|
82
|
+
return [title, " (none)"]
|
|
83
|
+
names = {column: tidy(column, settings.SHORT_NAME) for column in self.rules}
|
|
84
|
+
width = max(len(name) for name in names.values())
|
|
85
|
+
lines = [title]
|
|
86
|
+
for column, rule in self.rules.items():
|
|
87
|
+
lines.append(f" {names[column].ljust(width)} : {describe_rule(rule)}")
|
|
88
|
+
return lines
|
|
89
|
+
|
|
90
|
+
def notes_lines(self):
|
|
91
|
+
return ["Notes"] + [f" - {note}" for note in self.notes]
|
|
92
|
+
|
|
93
|
+
def table_lines(self):
|
|
94
|
+
header = ["File Row", "Column", "Problem", "Value", "Suggested fix"]
|
|
95
|
+
table = [header]
|
|
96
|
+
for e in self.errors:
|
|
97
|
+
row = "-" if e["row"] is None else str(e["row"])
|
|
98
|
+
table.append([
|
|
99
|
+
row,
|
|
100
|
+
tidy(e["column"], settings.SHORT_NAME),
|
|
101
|
+
tidy(e["reason"], settings.SHORT_TEXT),
|
|
102
|
+
tidy(e["value"], settings.SHORT_NAME),
|
|
103
|
+
tidy(e["suggestion"], settings.SHORT_TEXT),
|
|
104
|
+
])
|
|
105
|
+
widths = [max(len(r[i]) for r in table) for i in range(len(header))]
|
|
106
|
+
lines = []
|
|
107
|
+
for index, r in enumerate(table):
|
|
108
|
+
lines.append(" " + " | ".join(r[i].ljust(widths[i]) for i in range(len(header))).rstrip())
|
|
109
|
+
if index == 0:
|
|
110
|
+
lines.append(" " + "-+-".join("-" * w for w in widths))
|
|
111
|
+
return lines
|
|
112
|
+
|
|
113
|
+
def to_text(self):
|
|
114
|
+
lines = ["DataCheckup Report", f"File: {self.source}", ""]
|
|
115
|
+
lines += self.summary_lines()
|
|
116
|
+
lines += [""] + self.rules_lines()
|
|
117
|
+
if self.notes:
|
|
118
|
+
lines += [""] + self.notes_lines()
|
|
119
|
+
lines.append("")
|
|
120
|
+
if self.errors:
|
|
121
|
+
lines += self.table_lines()
|
|
122
|
+
else:
|
|
123
|
+
lines.append("No problems found.")
|
|
124
|
+
return "\n".join(lines)
|
|
125
|
+
|
|
126
|
+
def show(self):
|
|
127
|
+
print(self.to_text())
|
|
128
|
+
|
|
129
|
+
def export(self, path):
|
|
130
|
+
return export_problem_records(self.source, self.columns, self.records, self.errors, path)
|
datacheckup/rules.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import math
|
|
2
|
+
import numbers
|
|
3
|
+
import re
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
|
|
6
|
+
from . import messages, settings
|
|
7
|
+
from .errors import DataCheckupError
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def is_email(text):
|
|
11
|
+
return settings.EMAIL_PATTERN.match(text) is not None
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def is_phone(text):
|
|
15
|
+
digits = settings.PHONE_SEPARATORS.sub("", text)
|
|
16
|
+
return settings.PHONE_PATTERN.match(digits) is not None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def time_is_valid(hour, minute, second, meridian):
|
|
20
|
+
hour, minute = int(hour), int(minute)
|
|
21
|
+
if meridian and not 1 <= hour <= 12:
|
|
22
|
+
return False
|
|
23
|
+
if not meridian and hour > 23:
|
|
24
|
+
return False
|
|
25
|
+
if minute > 59:
|
|
26
|
+
return False
|
|
27
|
+
return second is None or int(second) <= 59
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def date_matches(text, style):
|
|
31
|
+
try:
|
|
32
|
+
datetime.strptime(text, style)
|
|
33
|
+
return True
|
|
34
|
+
except ValueError:
|
|
35
|
+
return False
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def is_date(text):
|
|
39
|
+
match = settings.DATE_TIME.match(text)
|
|
40
|
+
if match is None:
|
|
41
|
+
return False
|
|
42
|
+
date_part, hour, minute, second, meridian = match.groups()
|
|
43
|
+
if hour is not None and not time_is_valid(hour, minute, second, meridian):
|
|
44
|
+
return False
|
|
45
|
+
return any(date_matches(date_part, style) for style in settings.DATE_STYLES)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
FORMAT_CHECKS = {
|
|
49
|
+
"email": is_email,
|
|
50
|
+
"phone": is_phone,
|
|
51
|
+
"date": is_date,
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def clean_number_text(value):
|
|
56
|
+
text = settings.CURRENCY_MARKS.sub("", str(value))
|
|
57
|
+
text = settings.WHITESPACE.sub("", text)
|
|
58
|
+
if text.endswith("%"):
|
|
59
|
+
text = text[:-1]
|
|
60
|
+
return text
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def parse_number(value, kind="number"):
|
|
64
|
+
text = clean_number_text(value)
|
|
65
|
+
if not text or "_" in text:
|
|
66
|
+
return None
|
|
67
|
+
if "," in text:
|
|
68
|
+
if not settings.GROUPED_NUMBER.match(text):
|
|
69
|
+
return None
|
|
70
|
+
text = text.replace(",", "")
|
|
71
|
+
try:
|
|
72
|
+
number = float(text)
|
|
73
|
+
except ValueError:
|
|
74
|
+
return None
|
|
75
|
+
if not math.isfinite(number):
|
|
76
|
+
return None
|
|
77
|
+
if kind == "integer":
|
|
78
|
+
return int(number) if number.is_integer() else None
|
|
79
|
+
return number
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def check_limit_type(column, rule, limit):
|
|
83
|
+
if rule.get("type") not in settings.NUMBER_TYPES:
|
|
84
|
+
raise DataCheckupError(messages.LIMIT_NEEDS_NUMBER_TYPE.format(limit=limit, column=column))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def check_limit_value(column, rule, limit):
|
|
88
|
+
value = rule[limit]
|
|
89
|
+
if isinstance(value, bool) or not isinstance(value, numbers.Real):
|
|
90
|
+
raise DataCheckupError(messages.LIMIT_NOT_A_NUMBER.format(limit=limit, column=column))
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def check_pattern(column, rule):
|
|
94
|
+
pattern = rule["pattern"]
|
|
95
|
+
if not isinstance(pattern, str):
|
|
96
|
+
raise DataCheckupError(messages.PATTERN_NOT_TEXT.format(column=column))
|
|
97
|
+
try:
|
|
98
|
+
re.compile(pattern)
|
|
99
|
+
except re.error as error:
|
|
100
|
+
raise DataCheckupError(messages.PATTERN_INVALID.format(column=column, reason=error)) from None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def allowed_values(allowed):
|
|
104
|
+
if hasattr(allowed, "tolist"):
|
|
105
|
+
allowed = allowed.tolist()
|
|
106
|
+
if isinstance(allowed, (list, tuple, set, frozenset)):
|
|
107
|
+
return list(allowed)
|
|
108
|
+
return []
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def check_allowed(column, rule):
|
|
112
|
+
if not allowed_values(rule["allowed"]):
|
|
113
|
+
raise DataCheckupError(messages.ALLOWED_NOT_LIST.format(column=column))
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def check_column_rule(column, rule, partial=False):
|
|
117
|
+
if not isinstance(rule, dict):
|
|
118
|
+
raise DataCheckupError(messages.COLUMN_RULES_NOT_DICT.format(column=column))
|
|
119
|
+
unknown = set(rule) - settings.RULE_KEYS
|
|
120
|
+
if unknown:
|
|
121
|
+
raise DataCheckupError(messages.UNKNOWN_RULE.format(names=sorted(unknown), column=column))
|
|
122
|
+
if "type" in rule and rule["type"] not in settings.RULE_TYPES:
|
|
123
|
+
raise DataCheckupError(messages.UNKNOWN_TYPE.format(value=rule["type"], column=column))
|
|
124
|
+
if rule.get("format") is not None and rule["format"] not in FORMAT_CHECKS:
|
|
125
|
+
raise DataCheckupError(messages.UNKNOWN_FORMAT.format(value=rule["format"], column=column))
|
|
126
|
+
for limit in ("min", "max"):
|
|
127
|
+
if limit in rule:
|
|
128
|
+
if not partial:
|
|
129
|
+
check_limit_type(column, rule, limit)
|
|
130
|
+
check_limit_value(column, rule, limit)
|
|
131
|
+
if "pattern" in rule:
|
|
132
|
+
check_pattern(column, rule)
|
|
133
|
+
if "allowed" in rule:
|
|
134
|
+
check_allowed(column, rule)
|
|
135
|
+
if "message" in rule and not isinstance(rule["message"], str):
|
|
136
|
+
raise DataCheckupError(messages.MESSAGE_NOT_TEXT.format(column=column))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def load_rules(rules, partial=False):
|
|
140
|
+
if not isinstance(rules, dict):
|
|
141
|
+
raise DataCheckupError(messages.RULES_NOT_DICT)
|
|
142
|
+
cleaned = {}
|
|
143
|
+
for column, rule in rules.items():
|
|
144
|
+
check_column_rule(column, rule, partial)
|
|
145
|
+
cleaned[column] = dict(rule)
|
|
146
|
+
if "allowed" in rule:
|
|
147
|
+
cleaned[column]["allowed"] = allowed_values(rule["allowed"])
|
|
148
|
+
return cleaned
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def combine_rules(detected, custom, columns):
|
|
152
|
+
rules = {column: dict(rule) for column, rule in detected.items()}
|
|
153
|
+
notes = []
|
|
154
|
+
for column, rule in custom.items():
|
|
155
|
+
merged = rules.setdefault(column, {})
|
|
156
|
+
merged.update(rule)
|
|
157
|
+
if ("min" in merged or "max" in merged) and merged.get("type") is None:
|
|
158
|
+
merged["type"] = "number"
|
|
159
|
+
for limit in ("min", "max"):
|
|
160
|
+
if limit in merged:
|
|
161
|
+
check_limit_type(column, merged, limit)
|
|
162
|
+
if column not in columns and not rule.get("required"):
|
|
163
|
+
notes.append(messages.UNKNOWN_COLUMN_RULE.format(column=column, columns=", ".join(columns)))
|
|
164
|
+
if custom:
|
|
165
|
+
notes.append(messages.CUSTOM_RULES_APPLIED.format(columns=", ".join(custom)))
|
|
166
|
+
return rules, notes
|
datacheckup/settings.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
SUPPORTED_SUFFIXES = (".csv", ".json")
|
|
4
|
+
EXCEL_SUFFIXES = (".xlsx", ".xls", ".xlsm")
|
|
5
|
+
EXPORT_SUFFIXES = (".csv", ".json")
|
|
6
|
+
|
|
7
|
+
DELIMITERS = [",", ";", "\t", "|"]
|
|
8
|
+
ENCODINGS = ["utf-8-sig", "cp1252"]
|
|
9
|
+
|
|
10
|
+
UTF16_MARKS = (b"\xff\xfe", b"\xfe\xff")
|
|
11
|
+
EXCEL_SIGNATURE = b"PK\x03\x04"
|
|
12
|
+
PDF_SIGNATURE = b"%PDF"
|
|
13
|
+
TEXT_CONTROL_BYTES = (9, 10, 13)
|
|
14
|
+
|
|
15
|
+
HEAD_BYTES = 4096
|
|
16
|
+
READ_CHUNK = 1024 * 1024
|
|
17
|
+
HEADER_SEARCH_LINES = 100
|
|
18
|
+
BINARY_LIMIT = 0.05
|
|
19
|
+
|
|
20
|
+
ENTIRE_ROW = "(entire row)"
|
|
21
|
+
|
|
22
|
+
RULE_KEYS = {
|
|
23
|
+
"required", "unique", "type", "format", "min", "max",
|
|
24
|
+
"pattern", "allowed", "message", "sensitive",
|
|
25
|
+
}
|
|
26
|
+
RULE_TYPES = {"text", "integer", "number"}
|
|
27
|
+
NUMBER_TYPES = ("integer", "number")
|
|
28
|
+
SENSITIVE_FORMATS = {"email", "phone"}
|
|
29
|
+
|
|
30
|
+
DATE_STYLES = ["%Y-%m-%d", "%d-%m-%Y", "%d/%m/%Y", "%d.%m.%Y", "%Y/%m/%d"]
|
|
31
|
+
|
|
32
|
+
EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s.]+(\.[^@\s.]+)+$")
|
|
33
|
+
PHONE_PATTERN = re.compile(r"^\+?[0-9]{10,13}$")
|
|
34
|
+
PHONE_SEPARATORS = re.compile(r"[\s\-().]")
|
|
35
|
+
DATE_TIME = re.compile(
|
|
36
|
+
r"^([0-9]{1,4}[-/.][0-9]{1,2}[-/.][0-9]{1,4})"
|
|
37
|
+
r"(?:[ T]([0-9]{1,2}):([0-9]{2})(?::([0-9]{2})(?:\.[0-9]+)?)?\s?([AaPp][Mm])?"
|
|
38
|
+
r"\s?(?:Z|[+-][0-9]{2}:?[0-9]{2})?)?$"
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
CURRENCY_MARKS = re.compile(r"rs\.?|inr|[₹$€£]", re.IGNORECASE)
|
|
42
|
+
GROUPED_NUMBER = re.compile(r"^[+-]?[0-9]{1,3}(,[0-9]{2,3})+(\.[0-9]+)?$")
|
|
43
|
+
WHITESPACE = re.compile(r"\s+")
|
|
44
|
+
|
|
45
|
+
MIN_RECORDS = 5
|
|
46
|
+
MIN_VALUES = 3
|
|
47
|
+
AGREE = 0.9
|
|
48
|
+
NAME_MATCH = 0.5
|
|
49
|
+
DISTINCT = 0.9
|
|
50
|
+
SAMPLE_SIZE = 2000
|
|
51
|
+
|
|
52
|
+
NAME_HINTS = {
|
|
53
|
+
"email": {"email", "mail"},
|
|
54
|
+
"phone": {"phone", "mobile", "cell", "whatsapp", "telephone"},
|
|
55
|
+
"date": {"date", "dob", "birth"},
|
|
56
|
+
}
|
|
57
|
+
ID_WORDS = {"id", "no", "number", "num", "code", "roll", "sno"}
|
|
58
|
+
|
|
59
|
+
FORMULA_STARTS = ("=", "@", "\t", "\r")
|
|
60
|
+
SIGN_STARTS = ("+", "-")
|
|
61
|
+
|
|
62
|
+
SHORT_NAME = 40
|
|
63
|
+
SHORT_TEXT = 70
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datacheckup
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Find the exact row and column of every error in a CSV or JSON file, with no setup and no database.
|
|
5
|
+
Author: Corefix Group
|
|
6
|
+
Maintainer: Corefix Group
|
|
7
|
+
License: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/CoreFixGroup/DataCheckup
|
|
9
|
+
Project-URL: Repository, https://github.com/CoreFixGroup/DataCheckup
|
|
10
|
+
Project-URL: Issues, https://github.com/CoreFixGroup/DataCheckup/issues
|
|
11
|
+
Keywords: data quality,csv,json,validation,data cleaning
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# datacheckup
|
|
22
|
+
|
|
23
|
+
A small Python library that reads a CSV or JSON file and tells you the exact row and column of every error, with a plain-words reason and a fix. It finds the rules for each column by itself, lets you override only the ones you choose, and never changes your original file. It needs nothing but the Python standard library.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
pip install datacheckup
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Use
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from datacheckup import check
|
|
35
|
+
|
|
36
|
+
report = check("students.csv")
|
|
37
|
+
report.show()
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Override rules for chosen columns
|
|
41
|
+
|
|
42
|
+
Name only the columns you want to change. Every other column stays automatic.
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from datacheckup import check
|
|
46
|
+
|
|
47
|
+
rules = {
|
|
48
|
+
"Marks": {"min": 10, "max": 80},
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
report = check("students.csv", rules=rules)
|
|
52
|
+
report.show()
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Good to know
|
|
56
|
+
|
|
57
|
+
- Rules found automatically are smart guesses. Use your own rules for columns that matter.
|
|
58
|
+
- The whole file is loaded into memory, so very large files need a lot of RAM.
|
|
59
|
+
- Excel (.xlsx) files must be saved as CSV first.
|
|
60
|
+
- Requires Python 3.11 or newer.
|
|
61
|
+
|
|
62
|
+
## Licence
|
|
63
|
+
|
|
64
|
+
MIT
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
datacheckup/__init__.py,sha256=To8ve_VxVnx7hUTtciFHN0GhkmtD_NUZZkXlSJeQDLc,918
|
|
2
|
+
datacheckup/checks.py,sha256=r1OlNjbGO2wDglYbIbOoFuKSN4cPmegQ6b0cOjQFnO8,7256
|
|
3
|
+
datacheckup/detector.py,sha256=WJq-dAKbbtZIkZ7od5Xfwelz12dlm3-CfezzK04pki0,2734
|
|
4
|
+
datacheckup/errors.py,sha256=4ymMmr11AAqs6soMsPeo2v2uVFdLR-cQy8nlZv2rdFI,45
|
|
5
|
+
datacheckup/exporter.py,sha256=8AiNU7N-04YPWkh_lrtcg-qfabu1cJGn6yAOdkb_Rjw,2549
|
|
6
|
+
datacheckup/messages.py,sha256=CJyJ8Mb3bmzJIlmTSImfwFIgd5fsKm-Cs37XEPlyj3I,5691
|
|
7
|
+
datacheckup/reader.py,sha256=K6VGFas97vXl7TaAn-4hETvxQAN9n5yWQmSxhIgE3-M,6532
|
|
8
|
+
datacheckup/report.py,sha256=rSTM4ZAGCmmZsF0DRfEkxyJGPRDWy2o__9ABLWB_pPI,4574
|
|
9
|
+
datacheckup/rules.py,sha256=gFGsxHvrRcbvcF7QRs9J1WK53PlTqXm-7xlLUO2xGqw,5408
|
|
10
|
+
datacheckup/settings.py,sha256=fCpvVWvRD_j5zTi5PnHzBnaB5pEqURDmx90SnmmGAh8,1722
|
|
11
|
+
datacheckup-0.1.0.dist-info/licenses/LICENSE,sha256=8DzLorciLEc-HFOu4ww4RmYVnoLVi_1SGse13s8UdUQ,1069
|
|
12
|
+
datacheckup-0.1.0.dist-info/METADATA,sha256=Nl396lVyxf82h0j5ywWkqUBEaxOpgNpKcCxmmE2KRBE,1848
|
|
13
|
+
datacheckup-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
14
|
+
datacheckup-0.1.0.dist-info/top_level.txt,sha256=O3pchpWm5193XQbxHmkiD5GKNjBfbfQUkylDW9Mw-8M,12
|
|
15
|
+
datacheckup-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 CoreFixGroup
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
datacheckup
|