ke-data-clean 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ """ke-data-clean: clean Kenyan data without ever fixing anything silently."""
2
+
3
+ from ke_data_clean.result import Result, Status
4
+
5
+ __version__ = "0.1.0"
6
+
7
+ __all__ = ["Result", "Status", "__version__"]
ke_data_clean/cli.py ADDED
@@ -0,0 +1,281 @@
1
+ """The ke-clean command: a messy CSV in, a cleaned CSV and a report out.
2
+
3
+ Usage::
4
+
5
+ ke-clean FILE.csv --phone phone_no --pin kra_pin --county county \\
6
+ --amount total --out cleaned.csv --report changes.csv
7
+
8
+ Original columns are never touched: each cleaned column is written
9
+ beside them as ``<col>_clean`` and ``<col>_status``. Re-running the
10
+ command on its own output is safe and changes nothing: a derived
11
+ column is rewritten only when it already holds exactly the values
12
+ this run would write, so a column ke-clean did not produce is refused
13
+ rather than clobbered. The report lists every row that was fixed, was
14
+ invalid, or was ambiguous. With ``--strict`` the command exits
15
+ non-zero when any value stays invalid, so a pipeline can fail on it.
16
+
17
+ Exit codes: 0 when clean (or only warnings), 1 when ``--strict`` found
18
+ invalid values, 2 for usage and file errors.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import argparse
24
+ import os
25
+ import sys
26
+
27
+ import pandas as pd
28
+
29
+ from ke_data_clean import __version__
30
+ from ke_data_clean.kinds import KINDS
31
+ from ke_data_clean.pandas_ext import clean_column
32
+
33
+ _SUFFIX_CLEAN = "_clean"
34
+ _SUFFIX_STATUS = "_status"
35
+
36
+
37
+ def build_parser() -> argparse.ArgumentParser:
38
+ parser = argparse.ArgumentParser(
39
+ prog="ke-clean",
40
+ description="Clean Kenyan columns in a CSV: phones, KRA PINs, counties, KES amounts.",
41
+ )
42
+ parser.add_argument("file", help="the CSV file to clean")
43
+ parser.add_argument(
44
+ "--phone",
45
+ action="append",
46
+ default=[],
47
+ metavar="COL",
48
+ help="column holding phone numbers; repeat for more columns",
49
+ )
50
+ parser.add_argument(
51
+ "--pin",
52
+ action="append",
53
+ default=[],
54
+ metavar="COL",
55
+ help="column holding KRA PINs; repeat for more columns",
56
+ )
57
+ parser.add_argument(
58
+ "--county",
59
+ action="append",
60
+ default=[],
61
+ metavar="COL",
62
+ help="column holding county names; repeat for more columns",
63
+ )
64
+ parser.add_argument(
65
+ "--amount",
66
+ action="append",
67
+ default=[],
68
+ metavar="COL",
69
+ help="column holding KES amounts; repeat for more columns",
70
+ )
71
+ parser.add_argument("--out", required=True, help="path for the cleaned CSV")
72
+ parser.add_argument("--report", required=True, help="path for the change report CSV")
73
+ parser.add_argument(
74
+ "--allow-shorthand",
75
+ action="store_true",
76
+ help="accept 1.2k-style amounts (ambiguous in business data)",
77
+ )
78
+ parser.add_argument(
79
+ "--strict",
80
+ action="store_true",
81
+ help="exit 1 if any value remains invalid, so pipelines can fail on it",
82
+ )
83
+ parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
84
+ return parser
85
+
86
+
87
+ def _targets_from(args: argparse.Namespace) -> dict[str, list[str]]:
88
+ """Map each kind to the columns the user named for it."""
89
+ targets: dict[str, list[str]] = {}
90
+ for kind in KINDS:
91
+ columns = getattr(args, kind)
92
+ if columns:
93
+ targets[kind] = columns
94
+ return targets
95
+
96
+
97
+ def _check_columns_exist(frame: pd.DataFrame, targets: dict[str, list[str]]) -> str | None:
98
+ for columns in targets.values():
99
+ for column in columns:
100
+ if column not in frame.columns:
101
+ return (
102
+ f"column '{column}' not found in the input file. "
103
+ f"Columns present: {', '.join(frame.columns)}"
104
+ )
105
+ return None
106
+
107
+
108
+ def _same_path(left: str, right: str) -> bool:
109
+ """Whether two paths point at the same file, resolving links."""
110
+ return os.path.realpath(left) == os.path.realpath(right)
111
+
112
+
113
+ def _check_output_paths(args: argparse.Namespace) -> str | None:
114
+ for label, path in (("out", args.out), ("report", args.report)):
115
+ if _same_path(path, args.file):
116
+ return (
117
+ f"--{label} {path} is the input file itself; "
118
+ "writing it would destroy the original data"
119
+ )
120
+ return None
121
+
122
+
123
+ def _existing_column_matches(frame: pd.DataFrame, name: str, computed: pd.Series) -> bool:
124
+ """Whether ``name`` already holds exactly what this run would write.
125
+
126
+ Missing values compare equal to missing values, so a re-run of
127
+ ke-clean's own output (where invalid cells were written empty)
128
+ matches.
129
+ """
130
+ existing = frame[name].astype(object).where(frame[name].notna(), "")
131
+ computed = computed.astype(object).where(pd.notna(computed), "")
132
+ return bool(existing.equals(computed))
133
+
134
+
135
+ def _check_would_not_clobber(
136
+ frame: pd.DataFrame,
137
+ results: dict[str, pd.DataFrame],
138
+ ) -> str | None:
139
+ """Refuse to replace a derived column that ke-clean did not produce.
140
+
141
+ Re-running on ke-clean's own output must stay idempotent, so a
142
+ ``<col>_clean`` or ``<col>_status`` column whose values match the
143
+ recomputed ones is regenerated silently. A column that differs
144
+ holds foreign data, and overwriting it would destroy that data.
145
+ """
146
+ for column, result_frame in results.items():
147
+ for suffix, computed in (
148
+ (_SUFFIX_CLEAN, result_frame["value"]),
149
+ (_SUFFIX_STATUS, result_frame["status"]),
150
+ ):
151
+ name = column + suffix
152
+ if name in frame.columns and not _existing_column_matches(frame, name, computed):
153
+ return (
154
+ f"input already has a '{name}' column with values "
155
+ "ke-clean did not produce; overwriting it would destroy "
156
+ "data. Rename or drop it and run again"
157
+ )
158
+ return None
159
+
160
+
161
+ def _clean_all(
162
+ frame: pd.DataFrame,
163
+ targets: dict[str, list[str]],
164
+ allow_shorthand: bool,
165
+ ) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]]:
166
+ """Return the frame with _clean/_status columns and each column's results."""
167
+ cleaned = frame.copy()
168
+ results: dict[str, pd.DataFrame] = {}
169
+ for kind, columns in targets.items():
170
+ # Only the amount validator takes allow_shorthand; handing the
171
+ # flag to the other kinds would be a TypeError, not an ignore.
172
+ options: dict[str, object] = (
173
+ {"allow_shorthand": allow_shorthand} if kind == "amount" else {}
174
+ )
175
+ for column in columns:
176
+ result_frame = clean_column(cleaned[column], kind, **options)
177
+ results[column] = result_frame
178
+ cleaned[column + _SUFFIX_CLEAN] = result_frame["value"]
179
+ cleaned[column + _SUFFIX_STATUS] = result_frame["status"]
180
+ return cleaned, results
181
+
182
+
183
+ def _build_report(
184
+ original: pd.DataFrame,
185
+ results: dict[str, pd.DataFrame],
186
+ ) -> pd.DataFrame:
187
+ """One row per non-valid cell: row number, column, original, new, status, reason.
188
+
189
+ The columns are gathered as plain lists instead of row iteration:
190
+ ``iterrows`` builds a Series per row, which dominates the runtime on
191
+ six-figure row counts.
192
+ """
193
+ row_numbers: list[int] = []
194
+ columns: list[str] = []
195
+ originals: list[object] = []
196
+ new_values: list[object] = []
197
+ statuses: list[str] = []
198
+ reasons: list[str] = []
199
+ for column, result_frame in results.items():
200
+ changed = result_frame[result_frame["status"] != "valid"]
201
+ row_numbers.extend(position + 1 for position in changed.index.tolist())
202
+ columns.extend([column] * len(changed))
203
+ originals.extend(original.loc[changed.index, column].tolist())
204
+ new_values.extend(
205
+ value if value is not None else "" for value in changed["value"].tolist()
206
+ )
207
+ statuses.extend(changed["status"].tolist())
208
+ reasons.extend(changed["reason"].tolist())
209
+ return pd.DataFrame(
210
+ {
211
+ "row": row_numbers,
212
+ "column": columns,
213
+ "original": originals,
214
+ "new": new_values,
215
+ "status": statuses,
216
+ "reason": reasons,
217
+ },
218
+ dtype=object,
219
+ )
220
+
221
+
222
+ def _summarize(results: dict[str, pd.DataFrame]) -> str:
223
+ lines = []
224
+ statuses = ("valid", "fixed", "invalid", "ambiguous")
225
+ for column, result_frame in results.items():
226
+ counts = result_frame["status"].value_counts()
227
+ tally = ", ".join(f"{counts.get(status, 0)} {status}" for status in statuses)
228
+ lines.append(f"{column}: {tally}")
229
+ return "\n".join(lines)
230
+
231
+
232
+ def main(argv: list[str] | None = None) -> int:
233
+ """Entry point for the ke-clean console script."""
234
+ parser = build_parser()
235
+ args = parser.parse_args(argv)
236
+ try:
237
+ frame = pd.read_csv(args.file, dtype=str, keep_default_na=False, na_values=[""])
238
+ except (OSError, UnicodeDecodeError, pd.errors.ParserError) as error:
239
+ print(f"ke-clean: cannot read {args.file}: {error}", file=sys.stderr)
240
+ return 2
241
+ if not isinstance(frame.index, pd.RangeIndex):
242
+ # read_csv silently turns the first field into the index when a row
243
+ # has exactly one field more than the header. Refuse rather than
244
+ # clean shifted columns.
245
+ print(
246
+ f"ke-clean: cannot read {args.file} as a table: a data row has "
247
+ "more fields than the header; check quoting of values with commas",
248
+ file=sys.stderr,
249
+ )
250
+ return 2
251
+ targets = _targets_from(args)
252
+ if not targets:
253
+ print(
254
+ "ke-clean: name at least one column to clean, "
255
+ "for example --phone phone_no. Kinds: " + ", ".join(sorted(KINDS)),
256
+ file=sys.stderr,
257
+ )
258
+ return 2
259
+ checks = (_check_columns_exist(frame, targets), _check_output_paths(args))
260
+ for problem in checks:
261
+ if problem:
262
+ print(f"ke-clean: {problem}", file=sys.stderr)
263
+ return 2
264
+ cleaned, results = _clean_all(frame, targets, args.allow_shorthand)
265
+ clobber = _check_would_not_clobber(frame, results)
266
+ if clobber:
267
+ print(f"ke-clean: {clobber}", file=sys.stderr)
268
+ return 2
269
+ report = _build_report(frame, results)
270
+ cleaned.to_csv(args.out, index=False)
271
+ report.to_csv(args.report, index=False)
272
+ print(_summarize(results))
273
+ print(f"wrote {args.out} and {args.report} ({len(report)} report rows)")
274
+ if args.strict and (report["status"] == "invalid").any():
275
+ print("ke-clean: --strict: invalid values remain", file=sys.stderr)
276
+ return 1
277
+ return 0
278
+
279
+
280
+ if __name__ == "__main__": # pragma: no cover
281
+ raise SystemExit(main())
@@ -0,0 +1,161 @@
1
+ """County name and code normalization for all 47 counties of Kenya.
2
+
3
+ The data lives in ``data/counties.csv`` (code, name) and follows the
4
+ First Schedule of the Constitution of Kenya (2010): code 001 is Mombasa
5
+ and code 047 is Nairobi. Names are the official spellings with spaces
6
+ instead of hyphens or apostrophes except where the apostrophe is part of
7
+ the name (Murang'a).
8
+
9
+ Normalization steps: lowercase, strip apostrophes, hyphens and the word
10
+ "county", then exact alias lookup, then fuzzy matching with
11
+ ``difflib.get_close_matches``. When two or more candidates are equally
12
+ close, the result is ``ambiguous``: the tool refuses to guess.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import csv
18
+ import difflib
19
+ import importlib.resources
20
+ from functools import lru_cache
21
+
22
+ from ke_data_clean.result import Result
23
+
24
+ _FUZZY_CUTOFF = 0.75
25
+
26
+ _COUNTY_WORD = "county"
27
+
28
+
29
+ @lru_cache(maxsize=1)
30
+ def _load_rows() -> tuple[tuple[str, str], ...]:
31
+ """Read counties.csv once and return (code, name) pairs."""
32
+ resource = importlib.resources.files("ke_data_clean.data").joinpath("counties.csv")
33
+ with resource.open(newline="", encoding="utf-8") as handle:
34
+ return tuple((row["code"], row["name"]) for row in csv.DictReader(handle))
35
+
36
+
37
+ def _code_of(name: str) -> str:
38
+ for code, official in _load_rows():
39
+ if official == name:
40
+ return code
41
+ raise KeyError(name) # pragma: no cover - every name in ALIASES comes from the file
42
+
43
+
44
+ def _alias_map() -> dict[str, str]:
45
+ """Official names plus accepted spellings, keyed by a fuzzy key."""
46
+ keys: dict[str, str] = {}
47
+ for _code, official in _load_rows():
48
+ keys[_fuzzy_key(official)] = official
49
+ for alias, official in ALIASES.items():
50
+ keys.setdefault(_fuzzy_key(alias), official)
51
+ return keys
52
+
53
+
54
+ def _fuzzy_key(text: str) -> str:
55
+ """Lowercase and strip apostrophes, separators and the word 'county'."""
56
+ for character in ("'", "’"):
57
+ text = text.replace(character, "")
58
+ for character in ("-", "/", ",", "."):
59
+ text = text.replace(character, " ")
60
+ words = [word for word in text.lower().split() if word != _COUNTY_WORD]
61
+ return " ".join(words)
62
+
63
+
64
+ ALIASES: dict[str, str] = {
65
+ # Spellings seen in real datasets that differ beyond punctuation from
66
+ # the official name. Punctuation variants need no entry: _fuzzy_key
67
+ # already folds hyphens, slashes and apostrophes into the official key.
68
+ "Muranga": "Murang'a",
69
+ "Homabay": "Homa Bay",
70
+ "Nairobi City": "Nairobi",
71
+ "WestPokot": "West Pokot",
72
+ "ElgeyoMarakwet": "Elgeyo Marakwet",
73
+ "TaitaTaveta": "Taita Taveta",
74
+ "TharakaNithi": "Tharaka Nithi",
75
+ "Uasingishu": "Uasin Gishu",
76
+ }
77
+
78
+
79
+ def _official_names() -> list[str]:
80
+ return [name for _, name in _load_rows()]
81
+
82
+
83
+ def _exact_lookup(key: str) -> Result | None:
84
+ official = _alias_map().get(key)
85
+ if official is None:
86
+ return None
87
+ return Result(value=official, status="valid")
88
+
89
+
90
+ def _fuzzy_lookup(key: str) -> Result:
91
+ close = difflib.get_close_matches(key, list(_alias_map()), n=3, cutoff=_FUZZY_CUTOFF)
92
+ if not close:
93
+ return Result(value=None, status="invalid", reason="not a recognized county name")
94
+ # Several alias keys can map to the same official name; that is one
95
+ # candidate, not an ambiguity. Only distinct counties are ambiguous.
96
+ candidates: list[str] = []
97
+ for candidate_key in close:
98
+ official = _alias_map()[candidate_key]
99
+ if official not in candidates:
100
+ candidates.append(official)
101
+ if len(candidates) > 1:
102
+ return Result(
103
+ value=None,
104
+ status="ambiguous",
105
+ reason="could be any of: " + ", ".join(candidates),
106
+ )
107
+ return Result(value=candidates[0], status="fixed", reason="matched alias, fuzzy")
108
+
109
+
110
+ def normalize(raw: str) -> Result:
111
+ """Normalize one county name to the official name.
112
+
113
+ Args:
114
+ raw: The county name as found in the data, in any spelling.
115
+
116
+ Returns:
117
+ A Result whose value is the official county name when the status
118
+ is "valid" or "fixed", and None otherwise. Use ``code_for`` to
119
+ get the three-digit code.
120
+
121
+ Example:
122
+ >>> normalize("muranga")
123
+ Result(value="Murang'a", status='fixed', reason='matched alias, exact')
124
+ >>> normalize("kiifi").status
125
+ 'ambiguous'
126
+ """
127
+ if raw is None:
128
+ return Result(value=None, status="invalid", reason="no county name found")
129
+ text = str(raw).strip()
130
+ if not text:
131
+ return Result(value=None, status="invalid", reason="no county name found")
132
+ key = _fuzzy_key(text)
133
+ if not key:
134
+ return Result(value=None, status="invalid", reason="no county name found")
135
+ exact = _exact_lookup(key)
136
+ if exact is not None:
137
+ if text in _official_names():
138
+ return exact
139
+ return Result(value=exact.value, status="fixed", reason="matched alias, exact")
140
+ return _fuzzy_lookup(key)
141
+
142
+
143
+ def code_for(name: str) -> str | None:
144
+ """Return the three-digit county code for an official name, or None.
145
+
146
+ Example:
147
+ >>> code_for("Nairobi")
148
+ '047'
149
+ """
150
+ for code, official in _load_rows():
151
+ if official == name:
152
+ return code
153
+ return None
154
+
155
+
156
+ def all_counties() -> dict[str, str]:
157
+ """Return the full mapping of code to official name."""
158
+ return dict(_load_rows())
159
+
160
+
161
+ __all__ = ["all_counties", "code_for", "normalize"]
@@ -0,0 +1,3 @@
1
+ """Data files for ke-data-clean (counties.csv)."""
2
+
3
+ from __future__ import annotations
@@ -0,0 +1,48 @@
1
+ code,name
2
+ 001,Mombasa
3
+ 002,Kwale
4
+ 003,Kilifi
5
+ 004,Tana River
6
+ 005,Lamu
7
+ 006,Taita Taveta
8
+ 007,Garissa
9
+ 008,Wajir
10
+ 009,Mandera
11
+ 010,Marsabit
12
+ 011,Isiolo
13
+ 012,Meru
14
+ 013,Tharaka Nithi
15
+ 014,Embu
16
+ 015,Kitui
17
+ 016,Machakos
18
+ 017,Makueni
19
+ 018,Nyandarua
20
+ 019,Nyeri
21
+ 020,Kirinyaga
22
+ 021,Murang'a
23
+ 022,Kiambu
24
+ 023,Turkana
25
+ 024,West Pokot
26
+ 025,Samburu
27
+ 026,Trans Nzoia
28
+ 027,Uasin Gishu
29
+ 028,Elgeyo Marakwet
30
+ 029,Nandi
31
+ 030,Baringo
32
+ 031,Laikipia
33
+ 032,Nakuru
34
+ 033,Narok
35
+ 034,Kajiado
36
+ 035,Kericho
37
+ 036,Bomet
38
+ 037,Kakamega
39
+ 038,Vihiga
40
+ 039,Bungoma
41
+ 040,Busia
42
+ 041,Siaya
43
+ 042,Kisumu
44
+ 043,Homa Bay
45
+ 044,Migori
46
+ 045,Kisii
47
+ 046,Nyamira
48
+ 047,Nairobi
ke_data_clean/kinds.py ADDED
@@ -0,0 +1,48 @@
1
+ """The registry mapping kind names to the validator that handles each.
2
+
3
+ Both wrappers -- the pandas helper and the CLI -- dispatch through
4
+ ``clean_value`` so the same rules run everywhere. A kind is a word a
5
+ caller writes in their pipeline; the registry is the single place that
6
+ ties it to a function.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections.abc import Callable
12
+
13
+ from ke_data_clean import county, money, phone, pin
14
+ from ke_data_clean.result import Result
15
+
16
+ KINDS: dict[str, Callable[..., Result]] = {
17
+ "phone": phone.normalize,
18
+ "pin": pin.validate,
19
+ "county": county.normalize,
20
+ "amount": money.parse_kes,
21
+ }
22
+
23
+
24
+ def clean_value(raw: str, kind: str, **options: object) -> Result:
25
+ """Clean one raw value with the validator registered for ``kind``.
26
+
27
+ Args:
28
+ raw: The value as found in the data.
29
+ kind: One of "phone", "pin", "county", "amount".
30
+ **options: Forwarded to the validator, for example
31
+ ``allow_shorthand=True`` for amounts.
32
+
33
+ Raises:
34
+ ValueError: When the kind is not registered. The message lists
35
+ the known kinds.
36
+
37
+ Example:
38
+ >>> clean_value("0712 345 678", "phone").value
39
+ '+254712345678'
40
+ """
41
+ validator = KINDS.get(kind)
42
+ if validator is None:
43
+ known = ", ".join(sorted(KINDS))
44
+ raise ValueError(f"unknown kind '{kind}'. Known kinds: {known}")
45
+ return validator(raw, **options)
46
+
47
+
48
+ __all__ = ["KINDS", "clean_value"]
ke_data_clean/mask.py ADDED
@@ -0,0 +1,64 @@
1
+ """Masking helpers so Kenyan personal data can be shared more safely.
2
+
3
+ Phone numbers and ID numbers are personal data under Kenya's Data
4
+ Protection Act, 2019. Mask them before sharing extracts. These helpers
5
+ keep the digits an analyst needs to spot duplicates and hide the rest.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from ke_data_clean.phone import normalize as _normalize_phone
11
+ from ke_data_clean.result import Result
12
+
13
+
14
+ def phone(raw: str) -> Result:
15
+ """Mask a Kenyan phone number, keeping the first 6 and last 2 digits.
16
+
17
+ The number is normalized to E.164 first, so the masked form is
18
+ stable across input formats: ``0712 345 678`` and ``+254-712-345-678``
19
+ both give ``+25471*****78``. Invalid numbers return the phone
20
+ validator's result unchanged, so the reason travels with the data.
21
+
22
+ Example:
23
+ >>> phone("0712 345 678").value
24
+ '+25471*****78'
25
+ >>> phone("+254-712-345-678").value
26
+ '+25471*****78'
27
+ """
28
+ result = _normalize_phone(raw)
29
+ if result.value is None:
30
+ return result
31
+ digits = result.value
32
+ masked = digits[:6] + "*" * (len(digits) - 8) + digits[-2:]
33
+ return Result(value=masked, status=result.status, reason=result.reason)
34
+
35
+
36
+ def national_id(raw: str) -> Result:
37
+ """Mask a Kenyan national ID number, keeping the first 3 and last 3 digits.
38
+
39
+ The ID format is not verified: this is a masking helper, not a
40
+ validator. The mask is always three stars, so it does not reveal the
41
+ length of the ID. Fewer than 7 digits is refused: with the first and
42
+ last 3 kept, a shorter number would stay fully readable.
43
+
44
+ Example:
45
+ >>> national_id("12345678").value
46
+ '123***678'
47
+ >>> national_id("12345").status
48
+ 'invalid'
49
+ """
50
+ if raw is None:
51
+ return Result(value=None, status="invalid", reason="no ID number found")
52
+ digits = "".join(character for character in str(raw) if character.isdigit())
53
+ if not digits:
54
+ return Result(value=None, status="invalid", reason="no ID number found")
55
+ if len(digits) < 7:
56
+ return Result(value=None, status="invalid", reason="too short to mask safely")
57
+ return Result(
58
+ value=digits[:3] + "***" + digits[-3:],
59
+ status="fixed",
60
+ reason="masked for sharing",
61
+ )
62
+
63
+
64
+ __all__ = ["national_id", "phone"]