ke-data-clean 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ke_data_clean/__init__.py +7 -0
- ke_data_clean/cli.py +281 -0
- ke_data_clean/county.py +161 -0
- ke_data_clean/data/__init__.py +3 -0
- ke_data_clean/data/counties.csv +48 -0
- ke_data_clean/kinds.py +48 -0
- ke_data_clean/mask.py +64 -0
- ke_data_clean/money.py +154 -0
- ke_data_clean/mpesa.py +99 -0
- ke_data_clean/pandas_ext.py +75 -0
- ke_data_clean/phone.py +152 -0
- ke_data_clean/pin.py +99 -0
- ke_data_clean/py.typed +0 -0
- ke_data_clean/result.py +30 -0
- ke_data_clean-0.1.0.dist-info/METADATA +127 -0
- ke_data_clean-0.1.0.dist-info/RECORD +19 -0
- ke_data_clean-0.1.0.dist-info/WHEEL +4 -0
- ke_data_clean-0.1.0.dist-info/entry_points.txt +2 -0
- ke_data_clean-0.1.0.dist-info/licenses/LICENSE +21 -0
ke_data_clean/cli.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
"""The ke-clean command: a messy CSV in, a cleaned CSV and a report out.
|
|
2
|
+
|
|
3
|
+
Usage::
|
|
4
|
+
|
|
5
|
+
ke-clean FILE.csv --phone phone_no --pin kra_pin --county county \\
|
|
6
|
+
--amount total --out cleaned.csv --report changes.csv
|
|
7
|
+
|
|
8
|
+
Original columns are never touched: each cleaned column is written
|
|
9
|
+
beside them as ``<col>_clean`` and ``<col>_status``. Re-running the
|
|
10
|
+
command on its own output is safe and changes nothing: a derived
|
|
11
|
+
column is rewritten only when it already holds exactly the values
|
|
12
|
+
this run would write, so a column ke-clean did not produce is refused
|
|
13
|
+
rather than clobbered. The report lists every row that was fixed, was
|
|
14
|
+
invalid, or was ambiguous. With ``--strict`` the command exits
|
|
15
|
+
non-zero when any value stays invalid, so a pipeline can fail on it.
|
|
16
|
+
|
|
17
|
+
Exit codes: 0 when clean (or only warnings), 1 when ``--strict`` found
|
|
18
|
+
invalid values, 2 for usage and file errors.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import os
|
|
25
|
+
import sys
|
|
26
|
+
|
|
27
|
+
import pandas as pd
|
|
28
|
+
|
|
29
|
+
from ke_data_clean import __version__
|
|
30
|
+
from ke_data_clean.kinds import KINDS
|
|
31
|
+
from ke_data_clean.pandas_ext import clean_column
|
|
32
|
+
|
|
33
|
+
_SUFFIX_CLEAN = "_clean"
|
|
34
|
+
_SUFFIX_STATUS = "_status"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
38
|
+
parser = argparse.ArgumentParser(
|
|
39
|
+
prog="ke-clean",
|
|
40
|
+
description="Clean Kenyan columns in a CSV: phones, KRA PINs, counties, KES amounts.",
|
|
41
|
+
)
|
|
42
|
+
parser.add_argument("file", help="the CSV file to clean")
|
|
43
|
+
parser.add_argument(
|
|
44
|
+
"--phone",
|
|
45
|
+
action="append",
|
|
46
|
+
default=[],
|
|
47
|
+
metavar="COL",
|
|
48
|
+
help="column holding phone numbers; repeat for more columns",
|
|
49
|
+
)
|
|
50
|
+
parser.add_argument(
|
|
51
|
+
"--pin",
|
|
52
|
+
action="append",
|
|
53
|
+
default=[],
|
|
54
|
+
metavar="COL",
|
|
55
|
+
help="column holding KRA PINs; repeat for more columns",
|
|
56
|
+
)
|
|
57
|
+
parser.add_argument(
|
|
58
|
+
"--county",
|
|
59
|
+
action="append",
|
|
60
|
+
default=[],
|
|
61
|
+
metavar="COL",
|
|
62
|
+
help="column holding county names; repeat for more columns",
|
|
63
|
+
)
|
|
64
|
+
parser.add_argument(
|
|
65
|
+
"--amount",
|
|
66
|
+
action="append",
|
|
67
|
+
default=[],
|
|
68
|
+
metavar="COL",
|
|
69
|
+
help="column holding KES amounts; repeat for more columns",
|
|
70
|
+
)
|
|
71
|
+
parser.add_argument("--out", required=True, help="path for the cleaned CSV")
|
|
72
|
+
parser.add_argument("--report", required=True, help="path for the change report CSV")
|
|
73
|
+
parser.add_argument(
|
|
74
|
+
"--allow-shorthand",
|
|
75
|
+
action="store_true",
|
|
76
|
+
help="accept 1.2k-style amounts (ambiguous in business data)",
|
|
77
|
+
)
|
|
78
|
+
parser.add_argument(
|
|
79
|
+
"--strict",
|
|
80
|
+
action="store_true",
|
|
81
|
+
help="exit 1 if any value remains invalid, so pipelines can fail on it",
|
|
82
|
+
)
|
|
83
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
84
|
+
return parser
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _targets_from(args: argparse.Namespace) -> dict[str, list[str]]:
|
|
88
|
+
"""Map each kind to the columns the user named for it."""
|
|
89
|
+
targets: dict[str, list[str]] = {}
|
|
90
|
+
for kind in KINDS:
|
|
91
|
+
columns = getattr(args, kind)
|
|
92
|
+
if columns:
|
|
93
|
+
targets[kind] = columns
|
|
94
|
+
return targets
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _check_columns_exist(frame: pd.DataFrame, targets: dict[str, list[str]]) -> str | None:
|
|
98
|
+
for columns in targets.values():
|
|
99
|
+
for column in columns:
|
|
100
|
+
if column not in frame.columns:
|
|
101
|
+
return (
|
|
102
|
+
f"column '{column}' not found in the input file. "
|
|
103
|
+
f"Columns present: {', '.join(frame.columns)}"
|
|
104
|
+
)
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _same_path(left: str, right: str) -> bool:
|
|
109
|
+
"""Whether two paths point at the same file, resolving links."""
|
|
110
|
+
return os.path.realpath(left) == os.path.realpath(right)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _check_output_paths(args: argparse.Namespace) -> str | None:
|
|
114
|
+
for label, path in (("out", args.out), ("report", args.report)):
|
|
115
|
+
if _same_path(path, args.file):
|
|
116
|
+
return (
|
|
117
|
+
f"--{label} {path} is the input file itself; "
|
|
118
|
+
"writing it would destroy the original data"
|
|
119
|
+
)
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _existing_column_matches(frame: pd.DataFrame, name: str, computed: pd.Series) -> bool:
|
|
124
|
+
"""Whether ``name`` already holds exactly what this run would write.
|
|
125
|
+
|
|
126
|
+
Missing values compare equal to missing values, so a re-run of
|
|
127
|
+
ke-clean's own output (where invalid cells were written empty)
|
|
128
|
+
matches.
|
|
129
|
+
"""
|
|
130
|
+
existing = frame[name].astype(object).where(frame[name].notna(), "")
|
|
131
|
+
computed = computed.astype(object).where(pd.notna(computed), "")
|
|
132
|
+
return bool(existing.equals(computed))
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _check_would_not_clobber(
|
|
136
|
+
frame: pd.DataFrame,
|
|
137
|
+
results: dict[str, pd.DataFrame],
|
|
138
|
+
) -> str | None:
|
|
139
|
+
"""Refuse to replace a derived column that ke-clean did not produce.
|
|
140
|
+
|
|
141
|
+
Re-running on ke-clean's own output must stay idempotent, so a
|
|
142
|
+
``<col>_clean`` or ``<col>_status`` column whose values match the
|
|
143
|
+
recomputed ones is regenerated silently. A column that differs
|
|
144
|
+
holds foreign data, and overwriting it would destroy that data.
|
|
145
|
+
"""
|
|
146
|
+
for column, result_frame in results.items():
|
|
147
|
+
for suffix, computed in (
|
|
148
|
+
(_SUFFIX_CLEAN, result_frame["value"]),
|
|
149
|
+
(_SUFFIX_STATUS, result_frame["status"]),
|
|
150
|
+
):
|
|
151
|
+
name = column + suffix
|
|
152
|
+
if name in frame.columns and not _existing_column_matches(frame, name, computed):
|
|
153
|
+
return (
|
|
154
|
+
f"input already has a '{name}' column with values "
|
|
155
|
+
"ke-clean did not produce; overwriting it would destroy "
|
|
156
|
+
"data. Rename or drop it and run again"
|
|
157
|
+
)
|
|
158
|
+
return None
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _clean_all(
|
|
162
|
+
frame: pd.DataFrame,
|
|
163
|
+
targets: dict[str, list[str]],
|
|
164
|
+
allow_shorthand: bool,
|
|
165
|
+
) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]]:
|
|
166
|
+
"""Return the frame with _clean/_status columns and each column's results."""
|
|
167
|
+
cleaned = frame.copy()
|
|
168
|
+
results: dict[str, pd.DataFrame] = {}
|
|
169
|
+
for kind, columns in targets.items():
|
|
170
|
+
# Only the amount validator takes allow_shorthand; handing the
|
|
171
|
+
# flag to the other kinds would be a TypeError, not an ignore.
|
|
172
|
+
options: dict[str, object] = (
|
|
173
|
+
{"allow_shorthand": allow_shorthand} if kind == "amount" else {}
|
|
174
|
+
)
|
|
175
|
+
for column in columns:
|
|
176
|
+
result_frame = clean_column(cleaned[column], kind, **options)
|
|
177
|
+
results[column] = result_frame
|
|
178
|
+
cleaned[column + _SUFFIX_CLEAN] = result_frame["value"]
|
|
179
|
+
cleaned[column + _SUFFIX_STATUS] = result_frame["status"]
|
|
180
|
+
return cleaned, results
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _build_report(
|
|
184
|
+
original: pd.DataFrame,
|
|
185
|
+
results: dict[str, pd.DataFrame],
|
|
186
|
+
) -> pd.DataFrame:
|
|
187
|
+
"""One row per non-valid cell: row number, column, original, new, status, reason.
|
|
188
|
+
|
|
189
|
+
The columns are gathered as plain lists instead of row iteration:
|
|
190
|
+
``iterrows`` builds a Series per row, which dominates the runtime on
|
|
191
|
+
six-figure row counts.
|
|
192
|
+
"""
|
|
193
|
+
row_numbers: list[int] = []
|
|
194
|
+
columns: list[str] = []
|
|
195
|
+
originals: list[object] = []
|
|
196
|
+
new_values: list[object] = []
|
|
197
|
+
statuses: list[str] = []
|
|
198
|
+
reasons: list[str] = []
|
|
199
|
+
for column, result_frame in results.items():
|
|
200
|
+
changed = result_frame[result_frame["status"] != "valid"]
|
|
201
|
+
row_numbers.extend(position + 1 for position in changed.index.tolist())
|
|
202
|
+
columns.extend([column] * len(changed))
|
|
203
|
+
originals.extend(original.loc[changed.index, column].tolist())
|
|
204
|
+
new_values.extend(
|
|
205
|
+
value if value is not None else "" for value in changed["value"].tolist()
|
|
206
|
+
)
|
|
207
|
+
statuses.extend(changed["status"].tolist())
|
|
208
|
+
reasons.extend(changed["reason"].tolist())
|
|
209
|
+
return pd.DataFrame(
|
|
210
|
+
{
|
|
211
|
+
"row": row_numbers,
|
|
212
|
+
"column": columns,
|
|
213
|
+
"original": originals,
|
|
214
|
+
"new": new_values,
|
|
215
|
+
"status": statuses,
|
|
216
|
+
"reason": reasons,
|
|
217
|
+
},
|
|
218
|
+
dtype=object,
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _summarize(results: dict[str, pd.DataFrame]) -> str:
|
|
223
|
+
lines = []
|
|
224
|
+
statuses = ("valid", "fixed", "invalid", "ambiguous")
|
|
225
|
+
for column, result_frame in results.items():
|
|
226
|
+
counts = result_frame["status"].value_counts()
|
|
227
|
+
tally = ", ".join(f"{counts.get(status, 0)} {status}" for status in statuses)
|
|
228
|
+
lines.append(f"{column}: {tally}")
|
|
229
|
+
return "\n".join(lines)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def main(argv: list[str] | None = None) -> int:
|
|
233
|
+
"""Entry point for the ke-clean console script."""
|
|
234
|
+
parser = build_parser()
|
|
235
|
+
args = parser.parse_args(argv)
|
|
236
|
+
try:
|
|
237
|
+
frame = pd.read_csv(args.file, dtype=str, keep_default_na=False, na_values=[""])
|
|
238
|
+
except (OSError, UnicodeDecodeError, pd.errors.ParserError) as error:
|
|
239
|
+
print(f"ke-clean: cannot read {args.file}: {error}", file=sys.stderr)
|
|
240
|
+
return 2
|
|
241
|
+
if not isinstance(frame.index, pd.RangeIndex):
|
|
242
|
+
# read_csv silently turns the first field into the index when a row
|
|
243
|
+
# has exactly one field more than the header. Refuse rather than
|
|
244
|
+
# clean shifted columns.
|
|
245
|
+
print(
|
|
246
|
+
f"ke-clean: cannot read {args.file} as a table: a data row has "
|
|
247
|
+
"more fields than the header; check quoting of values with commas",
|
|
248
|
+
file=sys.stderr,
|
|
249
|
+
)
|
|
250
|
+
return 2
|
|
251
|
+
targets = _targets_from(args)
|
|
252
|
+
if not targets:
|
|
253
|
+
print(
|
|
254
|
+
"ke-clean: name at least one column to clean, "
|
|
255
|
+
"for example --phone phone_no. Kinds: " + ", ".join(sorted(KINDS)),
|
|
256
|
+
file=sys.stderr,
|
|
257
|
+
)
|
|
258
|
+
return 2
|
|
259
|
+
checks = (_check_columns_exist(frame, targets), _check_output_paths(args))
|
|
260
|
+
for problem in checks:
|
|
261
|
+
if problem:
|
|
262
|
+
print(f"ke-clean: {problem}", file=sys.stderr)
|
|
263
|
+
return 2
|
|
264
|
+
cleaned, results = _clean_all(frame, targets, args.allow_shorthand)
|
|
265
|
+
clobber = _check_would_not_clobber(frame, results)
|
|
266
|
+
if clobber:
|
|
267
|
+
print(f"ke-clean: {clobber}", file=sys.stderr)
|
|
268
|
+
return 2
|
|
269
|
+
report = _build_report(frame, results)
|
|
270
|
+
cleaned.to_csv(args.out, index=False)
|
|
271
|
+
report.to_csv(args.report, index=False)
|
|
272
|
+
print(_summarize(results))
|
|
273
|
+
print(f"wrote {args.out} and {args.report} ({len(report)} report rows)")
|
|
274
|
+
if args.strict and (report["status"] == "invalid").any():
|
|
275
|
+
print("ke-clean: --strict: invalid values remain", file=sys.stderr)
|
|
276
|
+
return 1
|
|
277
|
+
return 0
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
if __name__ == "__main__": # pragma: no cover
|
|
281
|
+
raise SystemExit(main())
|
ke_data_clean/county.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""County name and code normalization for all 47 counties of Kenya.
|
|
2
|
+
|
|
3
|
+
The data lives in ``data/counties.csv`` (code, name) and follows the
|
|
4
|
+
First Schedule of the Constitution of Kenya (2010): code 001 is Mombasa
|
|
5
|
+
and code 047 is Nairobi. Names are the official spellings with spaces
|
|
6
|
+
instead of hyphens or apostrophes except where the apostrophe is part of
|
|
7
|
+
the name (Murang'a).
|
|
8
|
+
|
|
9
|
+
Normalization steps: lowercase, strip apostrophes, hyphens and the word
|
|
10
|
+
"county", then exact alias lookup, then fuzzy matching with
|
|
11
|
+
``difflib.get_close_matches``. When two or more candidates are equally
|
|
12
|
+
close, the result is ``ambiguous``: the tool refuses to guess.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import csv
|
|
18
|
+
import difflib
|
|
19
|
+
import importlib.resources
|
|
20
|
+
from functools import lru_cache
|
|
21
|
+
|
|
22
|
+
from ke_data_clean.result import Result
|
|
23
|
+
|
|
24
|
+
_FUZZY_CUTOFF = 0.75
|
|
25
|
+
|
|
26
|
+
_COUNTY_WORD = "county"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@lru_cache(maxsize=1)
|
|
30
|
+
def _load_rows() -> tuple[tuple[str, str], ...]:
|
|
31
|
+
"""Read counties.csv once and return (code, name) pairs."""
|
|
32
|
+
resource = importlib.resources.files("ke_data_clean.data").joinpath("counties.csv")
|
|
33
|
+
with resource.open(newline="", encoding="utf-8") as handle:
|
|
34
|
+
return tuple((row["code"], row["name"]) for row in csv.DictReader(handle))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _code_of(name: str) -> str:
|
|
38
|
+
for code, official in _load_rows():
|
|
39
|
+
if official == name:
|
|
40
|
+
return code
|
|
41
|
+
raise KeyError(name) # pragma: no cover - every name in ALIASES comes from the file
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _alias_map() -> dict[str, str]:
|
|
45
|
+
"""Official names plus accepted spellings, keyed by a fuzzy key."""
|
|
46
|
+
keys: dict[str, str] = {}
|
|
47
|
+
for _code, official in _load_rows():
|
|
48
|
+
keys[_fuzzy_key(official)] = official
|
|
49
|
+
for alias, official in ALIASES.items():
|
|
50
|
+
keys.setdefault(_fuzzy_key(alias), official)
|
|
51
|
+
return keys
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _fuzzy_key(text: str) -> str:
|
|
55
|
+
"""Lowercase and strip apostrophes, separators and the word 'county'."""
|
|
56
|
+
for character in ("'", "’"):
|
|
57
|
+
text = text.replace(character, "")
|
|
58
|
+
for character in ("-", "/", ",", "."):
|
|
59
|
+
text = text.replace(character, " ")
|
|
60
|
+
words = [word for word in text.lower().split() if word != _COUNTY_WORD]
|
|
61
|
+
return " ".join(words)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
ALIASES: dict[str, str] = {
|
|
65
|
+
# Spellings seen in real datasets that differ beyond punctuation from
|
|
66
|
+
# the official name. Punctuation variants need no entry: _fuzzy_key
|
|
67
|
+
# already folds hyphens, slashes and apostrophes into the official key.
|
|
68
|
+
"Muranga": "Murang'a",
|
|
69
|
+
"Homabay": "Homa Bay",
|
|
70
|
+
"Nairobi City": "Nairobi",
|
|
71
|
+
"WestPokot": "West Pokot",
|
|
72
|
+
"ElgeyoMarakwet": "Elgeyo Marakwet",
|
|
73
|
+
"TaitaTaveta": "Taita Taveta",
|
|
74
|
+
"TharakaNithi": "Tharaka Nithi",
|
|
75
|
+
"Uasingishu": "Uasin Gishu",
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _official_names() -> list[str]:
|
|
80
|
+
return [name for _, name in _load_rows()]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _exact_lookup(key: str) -> Result | None:
|
|
84
|
+
official = _alias_map().get(key)
|
|
85
|
+
if official is None:
|
|
86
|
+
return None
|
|
87
|
+
return Result(value=official, status="valid")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _fuzzy_lookup(key: str) -> Result:
|
|
91
|
+
close = difflib.get_close_matches(key, list(_alias_map()), n=3, cutoff=_FUZZY_CUTOFF)
|
|
92
|
+
if not close:
|
|
93
|
+
return Result(value=None, status="invalid", reason="not a recognized county name")
|
|
94
|
+
# Several alias keys can map to the same official name; that is one
|
|
95
|
+
# candidate, not an ambiguity. Only distinct counties are ambiguous.
|
|
96
|
+
candidates: list[str] = []
|
|
97
|
+
for candidate_key in close:
|
|
98
|
+
official = _alias_map()[candidate_key]
|
|
99
|
+
if official not in candidates:
|
|
100
|
+
candidates.append(official)
|
|
101
|
+
if len(candidates) > 1:
|
|
102
|
+
return Result(
|
|
103
|
+
value=None,
|
|
104
|
+
status="ambiguous",
|
|
105
|
+
reason="could be any of: " + ", ".join(candidates),
|
|
106
|
+
)
|
|
107
|
+
return Result(value=candidates[0], status="fixed", reason="matched alias, fuzzy")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def normalize(raw: str) -> Result:
|
|
111
|
+
"""Normalize one county name to the official name.
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
raw: The county name as found in the data, in any spelling.
|
|
115
|
+
|
|
116
|
+
Returns:
|
|
117
|
+
A Result whose value is the official county name when the status
|
|
118
|
+
is "valid" or "fixed", and None otherwise. Use ``code_for`` to
|
|
119
|
+
get the three-digit code.
|
|
120
|
+
|
|
121
|
+
Example:
|
|
122
|
+
>>> normalize("muranga")
|
|
123
|
+
Result(value="Murang'a", status='fixed', reason='matched alias, exact')
|
|
124
|
+
>>> normalize("kiifi").status
|
|
125
|
+
'ambiguous'
|
|
126
|
+
"""
|
|
127
|
+
if raw is None:
|
|
128
|
+
return Result(value=None, status="invalid", reason="no county name found")
|
|
129
|
+
text = str(raw).strip()
|
|
130
|
+
if not text:
|
|
131
|
+
return Result(value=None, status="invalid", reason="no county name found")
|
|
132
|
+
key = _fuzzy_key(text)
|
|
133
|
+
if not key:
|
|
134
|
+
return Result(value=None, status="invalid", reason="no county name found")
|
|
135
|
+
exact = _exact_lookup(key)
|
|
136
|
+
if exact is not None:
|
|
137
|
+
if text in _official_names():
|
|
138
|
+
return exact
|
|
139
|
+
return Result(value=exact.value, status="fixed", reason="matched alias, exact")
|
|
140
|
+
return _fuzzy_lookup(key)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def code_for(name: str) -> str | None:
|
|
144
|
+
"""Return the three-digit county code for an official name, or None.
|
|
145
|
+
|
|
146
|
+
Example:
|
|
147
|
+
>>> code_for("Nairobi")
|
|
148
|
+
'047'
|
|
149
|
+
"""
|
|
150
|
+
for code, official in _load_rows():
|
|
151
|
+
if official == name:
|
|
152
|
+
return code
|
|
153
|
+
return None
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def all_counties() -> dict[str, str]:
|
|
157
|
+
"""Return the full mapping of code to official name."""
|
|
158
|
+
return dict(_load_rows())
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
__all__ = ["all_counties", "code_for", "normalize"]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
code,name
|
|
2
|
+
001,Mombasa
|
|
3
|
+
002,Kwale
|
|
4
|
+
003,Kilifi
|
|
5
|
+
004,Tana River
|
|
6
|
+
005,Lamu
|
|
7
|
+
006,Taita Taveta
|
|
8
|
+
007,Garissa
|
|
9
|
+
008,Wajir
|
|
10
|
+
009,Mandera
|
|
11
|
+
010,Marsabit
|
|
12
|
+
011,Isiolo
|
|
13
|
+
012,Meru
|
|
14
|
+
013,Tharaka Nithi
|
|
15
|
+
014,Embu
|
|
16
|
+
015,Kitui
|
|
17
|
+
016,Machakos
|
|
18
|
+
017,Makueni
|
|
19
|
+
018,Nyandarua
|
|
20
|
+
019,Nyeri
|
|
21
|
+
020,Kirinyaga
|
|
22
|
+
021,Murang'a
|
|
23
|
+
022,Kiambu
|
|
24
|
+
023,Turkana
|
|
25
|
+
024,West Pokot
|
|
26
|
+
025,Samburu
|
|
27
|
+
026,Trans Nzoia
|
|
28
|
+
027,Uasin Gishu
|
|
29
|
+
028,Elgeyo Marakwet
|
|
30
|
+
029,Nandi
|
|
31
|
+
030,Baringo
|
|
32
|
+
031,Laikipia
|
|
33
|
+
032,Nakuru
|
|
34
|
+
033,Narok
|
|
35
|
+
034,Kajiado
|
|
36
|
+
035,Kericho
|
|
37
|
+
036,Bomet
|
|
38
|
+
037,Kakamega
|
|
39
|
+
038,Vihiga
|
|
40
|
+
039,Bungoma
|
|
41
|
+
040,Busia
|
|
42
|
+
041,Siaya
|
|
43
|
+
042,Kisumu
|
|
44
|
+
043,Homa Bay
|
|
45
|
+
044,Migori
|
|
46
|
+
045,Kisii
|
|
47
|
+
046,Nyamira
|
|
48
|
+
047,Nairobi
|
ke_data_clean/kinds.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""The registry mapping kind names to the validator that handles each.
|
|
2
|
+
|
|
3
|
+
Both wrappers -- the pandas helper and the CLI -- dispatch through
|
|
4
|
+
``clean_value`` so the same rules run everywhere. A kind is a word a
|
|
5
|
+
caller writes in their pipeline; the registry is the single place that
|
|
6
|
+
ties it to a function.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
|
|
13
|
+
from ke_data_clean import county, money, phone, pin
|
|
14
|
+
from ke_data_clean.result import Result
|
|
15
|
+
|
|
16
|
+
KINDS: dict[str, Callable[..., Result]] = {
|
|
17
|
+
"phone": phone.normalize,
|
|
18
|
+
"pin": pin.validate,
|
|
19
|
+
"county": county.normalize,
|
|
20
|
+
"amount": money.parse_kes,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def clean_value(raw: str, kind: str, **options: object) -> Result:
|
|
25
|
+
"""Clean one raw value with the validator registered for ``kind``.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
raw: The value as found in the data.
|
|
29
|
+
kind: One of "phone", "pin", "county", "amount".
|
|
30
|
+
**options: Forwarded to the validator, for example
|
|
31
|
+
``allow_shorthand=True`` for amounts.
|
|
32
|
+
|
|
33
|
+
Raises:
|
|
34
|
+
ValueError: When the kind is not registered. The message lists
|
|
35
|
+
the known kinds.
|
|
36
|
+
|
|
37
|
+
Example:
|
|
38
|
+
>>> clean_value("0712 345 678", "phone").value
|
|
39
|
+
'+254712345678'
|
|
40
|
+
"""
|
|
41
|
+
validator = KINDS.get(kind)
|
|
42
|
+
if validator is None:
|
|
43
|
+
known = ", ".join(sorted(KINDS))
|
|
44
|
+
raise ValueError(f"unknown kind '{kind}'. Known kinds: {known}")
|
|
45
|
+
return validator(raw, **options)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
__all__ = ["KINDS", "clean_value"]
|
ke_data_clean/mask.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Masking helpers so Kenyan personal data can be shared more safely.
|
|
2
|
+
|
|
3
|
+
Phone numbers and ID numbers are personal data under Kenya's Data
|
|
4
|
+
Protection Act, 2019. Mask them before sharing extracts. These helpers
|
|
5
|
+
keep the digits an analyst needs to spot duplicates and hide the rest.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from ke_data_clean.phone import normalize as _normalize_phone
|
|
11
|
+
from ke_data_clean.result import Result
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def phone(raw: str) -> Result:
|
|
15
|
+
"""Mask a Kenyan phone number, keeping the first 6 and last 2 digits.
|
|
16
|
+
|
|
17
|
+
The number is normalized to E.164 first, so the masked form is
|
|
18
|
+
stable across input formats: ``0712 345 678`` and ``+254-712-345-678``
|
|
19
|
+
both give ``+25471*****78``. Invalid numbers return the phone
|
|
20
|
+
validator's result unchanged, so the reason travels with the data.
|
|
21
|
+
|
|
22
|
+
Example:
|
|
23
|
+
>>> phone("0712 345 678").value
|
|
24
|
+
'+25471*****78'
|
|
25
|
+
>>> phone("+254-712-345-678").value
|
|
26
|
+
'+25471*****78'
|
|
27
|
+
"""
|
|
28
|
+
result = _normalize_phone(raw)
|
|
29
|
+
if result.value is None:
|
|
30
|
+
return result
|
|
31
|
+
digits = result.value
|
|
32
|
+
masked = digits[:6] + "*" * (len(digits) - 8) + digits[-2:]
|
|
33
|
+
return Result(value=masked, status=result.status, reason=result.reason)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def national_id(raw: str) -> Result:
|
|
37
|
+
"""Mask a Kenyan national ID number, keeping the first 3 and last 3 digits.
|
|
38
|
+
|
|
39
|
+
The ID format is not verified: this is a masking helper, not a
|
|
40
|
+
validator. The mask is always three stars, so it does not reveal the
|
|
41
|
+
length of the ID. Fewer than 7 digits is refused: with the first and
|
|
42
|
+
last 3 kept, a shorter number would stay fully readable.
|
|
43
|
+
|
|
44
|
+
Example:
|
|
45
|
+
>>> national_id("12345678").value
|
|
46
|
+
'123***678'
|
|
47
|
+
>>> national_id("12345").status
|
|
48
|
+
'invalid'
|
|
49
|
+
"""
|
|
50
|
+
if raw is None:
|
|
51
|
+
return Result(value=None, status="invalid", reason="no ID number found")
|
|
52
|
+
digits = "".join(character for character in str(raw) if character.isdigit())
|
|
53
|
+
if not digits:
|
|
54
|
+
return Result(value=None, status="invalid", reason="no ID number found")
|
|
55
|
+
if len(digits) < 7:
|
|
56
|
+
return Result(value=None, status="invalid", reason="too short to mask safely")
|
|
57
|
+
return Result(
|
|
58
|
+
value=digits[:3] + "***" + digits[-3:],
|
|
59
|
+
status="fixed",
|
|
60
|
+
reason="masked for sharing",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
__all__ = ["national_id", "phone"]
|