cleansheet 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clean_sheet.py +605 -0
- clean_sheet_gui.py +193 -0
- clean_sheet_locales.py +94 -0
- cleansheet-1.1.0.dist-info/METADATA +173 -0
- cleansheet-1.1.0.dist-info/RECORD +9 -0
- cleansheet-1.1.0.dist-info/WHEEL +5 -0
- cleansheet-1.1.0.dist-info/entry_points.txt +3 -0
- cleansheet-1.1.0.dist-info/licenses/LICENSE +21 -0
- cleansheet-1.1.0.dist-info/top_level.txt +3 -0
clean_sheet.py
ADDED
|
@@ -0,0 +1,605 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""clean_sheet.py v2 - clean a messy CSV/XLSX table without silent guessing.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
python clean_sheet.py input.xlsx [--config cfg.json] [--out-dir DIR] [--sheet NAME] [--header-row 1] [--also-csv]
|
|
6
|
+
python clean_sheet.py input.csv --write-config cfg.json # save auto-detected column mapping to edit
|
|
7
|
+
|
|
8
|
+
Principles: the input file is never modified; every change is logged; anything ambiguous
|
|
9
|
+
or unparseable is flagged for human review, never silently guessed. Free libs only: openpyxl.
|
|
10
|
+
"""
|
|
11
|
+
import argparse, csv, difflib, json, re, sys, unicodedata
|
|
12
|
+
from collections import Counter, defaultdict
|
|
13
|
+
from datetime import date, datetime
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
import openpyxl
|
|
16
|
+
from openpyxl.styles import Font, PatternFill, Alignment
|
|
17
|
+
from openpyxl.utils import get_column_letter
|
|
18
|
+
from clean_sheet_locales import LOCALES
|
|
19
|
+
|
|
20
|
+
DEFAULTS = {
|
|
21
|
+
"columns": {}, "date_order": "dmy", "country_code": "213", "national_length": 9,
|
|
22
|
+
"mobile_prefixes": "23456789",
|
|
23
|
+
"leading_zero_required": True,
|
|
24
|
+
"currency_tokens": ["dzd", "da", "dinars", "dinar", "eur", "usd", "€", "$", "دينار جزائري", "دينار", "دنانير", "د.ج", "د ج", "دج"],
|
|
25
|
+
"category_maps": {}, "similarity_threshold": 0.88, "id_column": None,
|
|
26
|
+
}
|
|
27
|
+
NUM_WORDS = {"zero": 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7,
|
|
28
|
+
"eight": 8, "nine": 9, "ten": 10, "un": 1, "une": 1, "deux": 2, "trois": 3, "quatre": 4,
|
|
29
|
+
"cinq": 5, "sept": 7, "huit": 8, "neuf": 9, "dix": 10, "واحد": 1, "اثنان": 2, "ثلاثة": 3}
|
|
30
|
+
MONTHS = {m: i + 1 for i, names in enumerate([
|
|
31
|
+
["jan", "january", "janvier"], ["feb", "february", "fev", "fevrier", "février", "févr"],
|
|
32
|
+
["mar", "march", "mars"], ["apr", "april", "avr", "avril"], ["may", "mai"], ["jun", "june", "juin"],
|
|
33
|
+
["jul", "july", "juil", "juillet"], ["aug", "august", "aout", "août"],
|
|
34
|
+
["sep", "sept", "september", "septembre"], ["oct", "october", "octobre"],
|
|
35
|
+
["nov", "november", "novembre"], ["dec", "december", "decembre", "décembre"]]) for m in names}
|
|
36
|
+
_AR_STRIP = re.compile("[\u064b-\u065f\u0670\u0640]") # diacritics (tashkeel) and tatweel
|
|
37
|
+
_AR_MAP = str.maketrans({"أ": "ا", "إ": "ا", "آ": "ا", "ٱ": "ا", "ى": "ي", "ئ": "ي", "ؤ": "و", "ة": "ه"})
|
|
38
|
+
def ar_norm(s):
|
|
39
|
+
"""Normalise Arabic spelling variants for MATCHING only (hamza forms, ya/alef maqsura, ta marbuta, diacritics)."""
|
|
40
|
+
return _AR_STRIP.sub("", s).translate(_AR_MAP)
|
|
41
|
+
|
|
42
|
+
AR_DIGITS = str.maketrans("٠١٢٣٤٥٦٧٨٩۰۱۲۳۴۵۶۷۸۹٫٬", "0123456789" "0123456789" ".,")
|
|
43
|
+
TYPE_RULES_SRC = [
|
|
44
|
+
("email", r"e-?mail|courriel|بريد"),
|
|
45
|
+
("phone", r"phone|mobile|gsm|whatsapp|^tel\b|^tél\b|téléphone|هاتف|جوال|نقال|موبايل"),
|
|
46
|
+
("date", r"date|تاريخ"),
|
|
47
|
+
("qty", r"qty|quantit|qt[eé]?r?\b|qte|^nbr|الكمية|كمية|العدد|عدد"),
|
|
48
|
+
("price", r"price|prix|mo?n?g?tant|amount|tarif|p\.?u\b|سعر|ثمن|تمن|مبلغ|المبلغ|قيمة"),
|
|
49
|
+
("number", r"^(score|grade|moyenne|mark|marks|points?)\b|نقطة|النقطة|المعدل|العلامة"),
|
|
50
|
+
("id", r"order ?id|^id$|^ref\b|^réf|^n°|^no$|^num\b|id$|^رقم$|^رقم\s|المرجع"),
|
|
51
|
+
("category", r"city|ville|wilaya|product|produit|item|article|categor|catégor|status|statut|^mois\b|^month\b|مدينة|ولاية|بلدية|منتج|سلعة|صنف|فئة|الحالة|شهر"),
|
|
52
|
+
("name", r"customer|client|name|nom|prénom|prenom|société|company|زبون|اسم|لقب|عميل"),
|
|
53
|
+
]
|
|
54
|
+
TYPE_RULES = [(t, re.compile(ar_norm(rx))) for t, rx in TYPE_RULES_SRC]
|
|
55
|
+
FUZZY_WORDS = {"price": ["montant", "prix", "price", "amount", "tarif"], "qty": ["quantite", "quantity"]}
|
|
56
|
+
TYPES = {"id", "name", "category", "text", "qty", "price", "number", "date", "phone", "email", "skip"}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
MONTHS.update({ar_norm(k): v for k, v in {
|
|
60
|
+
"جانفي": 1, "فيفري": 2, "مارس": 3, "أفريل": 4, "ماي": 5, "جوان": 6, "جويلية": 7, "أوت": 8, "سبتمبر": 9, "أكتوبر": 10, "نوفمبر": 11, "ديسمبر": 12,
|
|
61
|
+
"يناير": 1, "فبراير": 2, "أبريل": 4, "مايو": 5, "يونيو": 6, "يوليو": 7, "أغسطس": 8,
|
|
62
|
+
"كانون الثاني": 1, "شباط": 2, "آذار": 3, "نيسان": 4, "أيار": 5, "حزيران": 6, "تموز": 7, "آب": 8, "أيلول": 9,
|
|
63
|
+
"تشرين الأول": 10, "تشرين الثاني": 11, "كانون الأول": 12,
|
|
64
|
+
"دجنبر": 12}.items()})
|
|
65
|
+
NUM_WORDS.update({ar_norm(k): v for k, v in {"واحد": 1, "واحدة": 1, "اثنان": 2, "اثنين": 2, "ثلاثة": 3, "أربعة": 4, "خمسة": 5, "ستة": 6,
|
|
66
|
+
"سبعة": 7, "ثمانية": 8, "تسعة": 9, "عشرة": 10}.items()})
|
|
67
|
+
|
|
68
|
+
def norm_text(v):
|
|
69
|
+
if v is None: return ""
|
|
70
|
+
s = unicodedata.normalize("NFC", str(v)).translate(AR_DIGITS)
|
|
71
|
+
s = re.sub(r"[\u00a0\u202f\u200e\u200f\u200b]", " ", s)
|
|
72
|
+
return re.sub(r"\s+", " ", s).strip()
|
|
73
|
+
|
|
74
|
+
def norm_num(v):
|
|
75
|
+
return "" if v is None else norm_text(str(v).translate(AR_DIGITS))
|
|
76
|
+
|
|
77
|
+
def detect_type(h):
|
|
78
|
+
x = ar_norm(str(h).casefold().strip())
|
|
79
|
+
for t, rx in TYPE_RULES:
|
|
80
|
+
if rx.search(x): return t
|
|
81
|
+
for w in re.findall(r"[^\W\d_]{4,}", x): # tolerate typos such as "mongtant"
|
|
82
|
+
for t, words in FUZZY_WORDS.items():
|
|
83
|
+
if any(difflib.SequenceMatcher(None, w, k).ratio() >= 0.8 for k in words): return t
|
|
84
|
+
return "text"
|
|
85
|
+
|
|
86
|
+
# ---------- parsers: return (value, status, message). status: ok | verify | fail | missing ----------
|
|
87
|
+
def parse_qty(v, cfg):
|
|
88
|
+
if isinstance(v, bool): return v, "fail", "unexpected boolean"
|
|
89
|
+
if isinstance(v, (int, float)):
|
|
90
|
+
if float(v).is_integer() and v >= 0: return int(v), "ok", ""
|
|
91
|
+
return v, "fail", "quantity is not a whole non-negative number"
|
|
92
|
+
s = norm_num(v).casefold()
|
|
93
|
+
if not s: return None, "missing", "quantity missing"
|
|
94
|
+
if re.fullmatch(r"\d+", s): return int(s), "ok", ""
|
|
95
|
+
if re.fullmatch(r"\d+[.,]0+", s): return int(re.split(r"[.,]", s)[0]), "ok", ""
|
|
96
|
+
if ar_norm(s) in NUM_WORDS: return NUM_WORDS[ar_norm(s)], "ok", ""
|
|
97
|
+
return v, "fail", "could not read quantity"
|
|
98
|
+
|
|
99
|
+
def parse_price(v, cfg):
|
|
100
|
+
if isinstance(v, bool): return v, "fail", "unexpected boolean"
|
|
101
|
+
if isinstance(v, (int, float)):
|
|
102
|
+
if v < 0: return float(v), "verify", "negative price"
|
|
103
|
+
return float(v), "ok", ""
|
|
104
|
+
s = norm_num(v)
|
|
105
|
+
if not s: return None, "missing", "price missing"
|
|
106
|
+
t = s
|
|
107
|
+
for tok in sorted(cfg["currency_tokens"], key=len, reverse=True):
|
|
108
|
+
t = re.sub(re.escape(tok), "", t, flags=re.I)
|
|
109
|
+
t = t.strip().replace("'", "")
|
|
110
|
+
if not re.fullmatch(r"-?[\d.,\s]+", t or "x"): return v, "fail", "could not read price"
|
|
111
|
+
neg = t.startswith("-"); t = t.lstrip("-").strip()
|
|
112
|
+
if " " in t:
|
|
113
|
+
if not re.fullmatch(r"\d{1,3}( \d{3})+([.,]\d{1,2})?", t): return v, "fail", "odd spacing in price"
|
|
114
|
+
t = t.replace(" ", "")
|
|
115
|
+
msg, status = "", "ok"
|
|
116
|
+
if "," in t and "." in t:
|
|
117
|
+
dec = "," if t.rfind(",") > t.rfind(".") else "."
|
|
118
|
+
thou = "." if dec == "," else ","
|
|
119
|
+
t = t.replace(thou, "").replace(dec, ".")
|
|
120
|
+
elif "," in t or "." in t:
|
|
121
|
+
sep = "," if "," in t else "."
|
|
122
|
+
parts = t.split(sep)
|
|
123
|
+
if len(parts) > 2:
|
|
124
|
+
if all(len(p) == 3 for p in parts[1:]) and 1 <= len(parts[0]) <= 3: t = "".join(parts)
|
|
125
|
+
else: return v, "fail", "inconsistent separators in price"
|
|
126
|
+
else:
|
|
127
|
+
after = parts[1]
|
|
128
|
+
if len(after) == 3 and 1 <= len(parts[0]) <= 3:
|
|
129
|
+
t = "".join(parts); status = "verify"
|
|
130
|
+
msg = f"'{s}' is ambiguous (thousands vs decimal); read as {t}"
|
|
131
|
+
elif len(after) in (1, 2): t = parts[0] + "." + after
|
|
132
|
+
else: return v, "fail", "could not read decimals in price"
|
|
133
|
+
try: val = float(t)
|
|
134
|
+
except ValueError: return v, "fail", "could not read price"
|
|
135
|
+
if neg: val = -val; status, msg = "verify", "negative price"
|
|
136
|
+
if val == 0: status, msg = "verify", "price is zero"
|
|
137
|
+
return val, status, msg
|
|
138
|
+
|
|
139
|
+
def _mkdate(y, m, d):
|
|
140
|
+
try: return date(y, m, d)
|
|
141
|
+
except ValueError: return None
|
|
142
|
+
|
|
143
|
+
def parse_date(v, cfg):
|
|
144
|
+
if isinstance(v, datetime): return v.date(), "ok", ""
|
|
145
|
+
if isinstance(v, date): return v, "ok", ""
|
|
146
|
+
s = norm_num(v)
|
|
147
|
+
if not s: return None, "missing", "date missing"
|
|
148
|
+
m = re.fullmatch(r"(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})(?:[ T]\d{1,2}:\d{2}(?::\d{2})?)?", s)
|
|
149
|
+
if m:
|
|
150
|
+
d = _mkdate(int(m[1]), int(m[2]), int(m[3]))
|
|
151
|
+
return (d, "ok", "") if d else (v, "fail", "impossible date")
|
|
152
|
+
m = re.fullmatch(r"(\d{1,2})[-/.](\d{1,2})[-/.](\d{2}|\d{4})", s)
|
|
153
|
+
if m:
|
|
154
|
+
a, b, y = int(m[1]), int(m[2]), int(m[3]); status, msg = "ok", ""
|
|
155
|
+
if y < 100: y += 2000; status, msg = "verify", "two-digit year read as 20xx"
|
|
156
|
+
if a > 12 and b <= 12: day, mon = a, b
|
|
157
|
+
elif b > 12 and a <= 12: day, mon = b, a
|
|
158
|
+
elif a == b: day, mon = a, b
|
|
159
|
+
elif a <= 12 and b <= 12:
|
|
160
|
+
day, mon = (a, b) if cfg["date_order"] == "dmy" else (b, a)
|
|
161
|
+
status, msg = "verify", f"ambiguous day/month in '{s}'; read as {cfg['date_order'].upper()}"
|
|
162
|
+
else: return v, "fail", "impossible date"
|
|
163
|
+
d = _mkdate(y, mon, day)
|
|
164
|
+
return (d, status, msg) if d else (v, "fail", "impossible date")
|
|
165
|
+
s = ar_norm(s)
|
|
166
|
+
for rx, order in ((r"(\d{1,2})\s+([^\W\d_]+(?:\s[^\W\d_]+)?)\.?,?\s+(\d{4})", "dmy"), (r"([^\W\d_]+(?:\s[^\W\d_]+)?)\.?\s+(\d{1,2}),?\s+(\d{4})", "mdy")):
|
|
167
|
+
m = re.fullmatch(rx, s)
|
|
168
|
+
if m:
|
|
169
|
+
day, mon, y = (m[1], m[2], m[3]) if order == "dmy" else (m[2], m[1], m[3])
|
|
170
|
+
mo = MONTHS.get(mon.casefold())
|
|
171
|
+
d = _mkdate(int(y), mo, int(day)) if mo else None
|
|
172
|
+
return (d, "ok", "") if d else (v, "fail", "could not read date")
|
|
173
|
+
return v, "fail", "could not read date"
|
|
174
|
+
|
|
175
|
+
def parse_phone(v, cfg):
|
|
176
|
+
s = norm_num(v)
|
|
177
|
+
if not s: return None, "missing", "phone missing"
|
|
178
|
+
cc, nl = cfg["country_code"], cfg["national_length"]
|
|
179
|
+
digits = re.sub(r"\D", "", s)
|
|
180
|
+
intl = s.startswith("+") or s.startswith("00")
|
|
181
|
+
if s.startswith("00"): digits = digits[2:]
|
|
182
|
+
status, msg = "ok", ""
|
|
183
|
+
if not cc: # generic locale: no country assumed, only plausibility
|
|
184
|
+
if len(digits) < 8 or len(digits) > 15: return v, "fail", f"phone has {len(digits)} digits, expected between 8 and 15"
|
|
185
|
+
if intl: return ("+" if s.startswith("+") else "") + digits, "ok", ""
|
|
186
|
+
return digits, "verify", "no country code set (--locale); number kept as written"
|
|
187
|
+
if intl:
|
|
188
|
+
if digits.startswith(cc): nat = digits[len(cc):]
|
|
189
|
+
elif 8 <= len(digits) <= 15: return "+" + digits, "ok", ""
|
|
190
|
+
else: return v, "fail", "could not read phone number"
|
|
191
|
+
if nat.startswith("0"): nat = nat[1:]
|
|
192
|
+
elif digits.startswith("0"): nat = digits[1:]
|
|
193
|
+
elif digits.startswith(cc) and len(digits) == len(cc) + nl: nat = digits[len(cc):]
|
|
194
|
+
else:
|
|
195
|
+
nat = digits
|
|
196
|
+
if len(nat) == nl and cfg.get("leading_zero_required", True):
|
|
197
|
+
status, msg = "verify", "leading 0 missing in source; assumed local number"
|
|
198
|
+
if len(nat) != nl: return v, "fail", f"phone has {len(nat)} digits after country code, expected {nl}"
|
|
199
|
+
prefixes = cfg.get("mobile_prefixes") or ""
|
|
200
|
+
if prefixes and nat[0] not in prefixes: status, msg = "verify", f"unusual +{cc} number prefix"
|
|
201
|
+
return f"+{cc}{nat}", status, msg
|
|
202
|
+
|
|
203
|
+
def parse_email(v, cfg):
|
|
204
|
+
s = norm_text(v).lower()
|
|
205
|
+
if not s: return None, "missing", "email missing"
|
|
206
|
+
if re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]{2,}", s): return s, "ok", ""
|
|
207
|
+
return v, "fail", "not a valid email format"
|
|
208
|
+
|
|
209
|
+
def parse_number(v, cfg):
|
|
210
|
+
"""Generic number (grade, score, measure): Arabic/Persian digits and decimal separators converted; ambiguous separators flagged."""
|
|
211
|
+
val, st, msg = parse_price(v, cfg)
|
|
212
|
+
if st in ("missing", "fail"): return val, st, msg.replace("price", "number")
|
|
213
|
+
if msg in ("price is zero", "negative price"): st, msg = "ok", ""
|
|
214
|
+
if isinstance(val, float) and val.is_integer(): val = int(val)
|
|
215
|
+
return val, st, msg
|
|
216
|
+
|
|
217
|
+
PARSERS = {"qty": parse_qty, "price": parse_price, "number": parse_number, "date": parse_date, "phone": parse_phone, "email": parse_email}
|
|
218
|
+
|
|
219
|
+
def same(orig, new):
|
|
220
|
+
if orig is None: return new in (None, "")
|
|
221
|
+
if isinstance(new, date): return isinstance(orig, (date, datetime)) or str(orig).strip() == new.isoformat()
|
|
222
|
+
if isinstance(new, (int, float)) and not isinstance(new, bool):
|
|
223
|
+
if isinstance(orig, (int, float)): return float(orig) == float(new)
|
|
224
|
+
if any(ch.isdigit() and not ch.isascii() for ch in str(orig)): return False
|
|
225
|
+
try: return float(str(orig).strip()) == float(new)
|
|
226
|
+
except ValueError: return False
|
|
227
|
+
return str(orig).strip() == str(new)
|
|
228
|
+
|
|
229
|
+
def proper(v):
|
|
230
|
+
return v.lower() == v.upper() or (v != v.lower() and v != v.upper())
|
|
231
|
+
|
|
232
|
+
def clean_text_column(raw, cfg, header, kind):
|
|
233
|
+
maps = {k.casefold(): v for k, v in cfg["category_maps"].get(header, {}).items()}
|
|
234
|
+
native = [isinstance(r, (int, float, datetime, date)) and not isinstance(r, bool) for r in raw] # numbers/dates are never turned into text
|
|
235
|
+
vals = []
|
|
236
|
+
for r, nat in zip(raw, native):
|
|
237
|
+
n = "" if nat else norm_text(r)
|
|
238
|
+
if n and n.casefold() in maps: n = maps[n.casefold()]
|
|
239
|
+
vals.append(n)
|
|
240
|
+
freq = Counter(v for v in vals if v)
|
|
241
|
+
groups = defaultdict(list)
|
|
242
|
+
for v in freq: groups[v.casefold()].append(v)
|
|
243
|
+
canon = {}
|
|
244
|
+
for variants in groups.values():
|
|
245
|
+
good = [x for x in variants if proper(x)]
|
|
246
|
+
pick = max(good, key=lambda x: freq[x]) if good else max(variants, key=lambda x: freq[x]).title()
|
|
247
|
+
for x in variants: canon[x] = pick
|
|
248
|
+
cfreq = Counter()
|
|
249
|
+
for v, c in freq.items(): cfreq[canon[v]] += c
|
|
250
|
+
key = lambda x: re.sub(r"[\W_]+", "", ar_norm(x.casefold()))
|
|
251
|
+
flagged = {}
|
|
252
|
+
if 1 < len(cfreq) <= 300:
|
|
253
|
+
items = sorted(cfreq, key=lambda x: -cfreq[x])
|
|
254
|
+
for i, a_ in enumerate(items):
|
|
255
|
+
for b_ in items[:i]:
|
|
256
|
+
same_key = key(a_) == key(b_)
|
|
257
|
+
if same_key or difflib.SequenceMatcher(None, key(a_), key(b_)).ratio() >= cfg["similarity_threshold"]:
|
|
258
|
+
flagged[a_] = f"may be the same as '{b_}' ({'differs only by punctuation, spacing or Arabic letter variants' if same_key else 'similar spelling'}); not merged"
|
|
259
|
+
break
|
|
260
|
+
out = []
|
|
261
|
+
for r, v, nat in zip(raw, vals, native):
|
|
262
|
+
if nat: out.append((r, "ok", "")); continue
|
|
263
|
+
if not v:
|
|
264
|
+
out.append((None, "missing" if kind != "text" else "ok", "value missing" if kind != "text" else "")); continue
|
|
265
|
+
c = canon[v]
|
|
266
|
+
out.append((c, "verify", flagged[c]) if c in flagged else (c, "ok", ""))
|
|
267
|
+
return out
|
|
268
|
+
|
|
269
|
+
def sniff_type(values, cfg):
|
|
270
|
+
"""Guess a column type from its CONTENT (used only when the header gives no clue)."""
|
|
271
|
+
vals = [v for v in values if norm_text(v) != ""][:200]
|
|
272
|
+
if len(vals) < 2: return "text"
|
|
273
|
+
frac = lambda f: sum(1 for v in vals if f(v)) / len(vals)
|
|
274
|
+
if frac(lambda v: isinstance(v, str) and re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]{2,}", norm_text(v).lower())) >= 0.8: return "email"
|
|
275
|
+
def isphone(v):
|
|
276
|
+
t = re.sub(r"[\s().\-]", "", norm_num(v))
|
|
277
|
+
return bool(re.fullmatch(r"(\+|00)\d{8,14}|0\d{8,10}|[567]\d{8}", t))
|
|
278
|
+
if frac(isphone) >= 0.8: return "phone"
|
|
279
|
+
if frac(lambda v: isinstance(v, (datetime, date)) or (isinstance(v, str) and parse_date(v, cfg)[1] in ("ok", "verify"))) >= 0.8: return "date"
|
|
280
|
+
toks = cfg["currency_tokens"]
|
|
281
|
+
if frac(lambda v: isinstance(v, str) and any(re.search(re.escape(t), norm_num(v), re.I) for t in toks) and parse_price(v, cfg)[1] in ("ok", "verify")) >= 0.6: return "price"
|
|
282
|
+
def isnum(v):
|
|
283
|
+
if isinstance(v, bool): return False
|
|
284
|
+
if isinstance(v, (int, float)): return True
|
|
285
|
+
t = norm_num(v)
|
|
286
|
+
return bool(re.fullmatch(r"-?\d+([.,]\d+)?", t)) and not re.fullmatch(r"0\d+", t)
|
|
287
|
+
if frac(isnum) >= 0.8: return "number"
|
|
288
|
+
return "text"
|
|
289
|
+
|
|
290
|
+
def merge_locale(user_cfg, locale):
|
|
291
|
+
"""Layer a locale pack under the user's config; explicit --config keys win."""
|
|
292
|
+
loc = LOCALES.get(locale or "")
|
|
293
|
+
if not loc: return user_cfg
|
|
294
|
+
merged = {k: v for k, v in loc.items() if k != "months"}
|
|
295
|
+
merged.update(user_cfg)
|
|
296
|
+
MONTHS.update({ar_norm(k): v for k, v in loc.get("months", {}).items()})
|
|
297
|
+
return merged
|
|
298
|
+
|
|
299
|
+
def resolve_config(user_cfg, headers, rows=None, detect_names=None):
|
|
300
|
+
cfg = json.loads(json.dumps(DEFAULTS)); cfg.update({k: v for k, v in (user_cfg or {}).items() if k in DEFAULTS})
|
|
301
|
+
declared = dict(cfg["columns"]); auto = {}
|
|
302
|
+
for j, h in enumerate(headers):
|
|
303
|
+
if h in declared:
|
|
304
|
+
if declared[h] not in TYPES: raise SystemExit(f"Unknown type '{declared[h]}' for column '{h}'. Allowed: {sorted(TYPES)}")
|
|
305
|
+
continue
|
|
306
|
+
t = detect_type((detect_names or {}).get(h, h))
|
|
307
|
+
if rows:
|
|
308
|
+
vals = [r["raw"][j] for r in rows]; nonempty = [v for v in vals if norm_text(v) != ""]
|
|
309
|
+
if t in ("name", "category") and nonempty and sum(1 for v in nonempty if isinstance(v, (int, float)) and not isinstance(v, bool)) / len(nonempty) >= 0.9: t = "number"
|
|
310
|
+
elif t == "text": t = sniff_type(vals, cfg)
|
|
311
|
+
auto[h] = t
|
|
312
|
+
cfg["columns"] = {**auto, **{h: t for h, t in declared.items() if h in headers}}
|
|
313
|
+
cfg["_auto_detected"] = sorted(auto)
|
|
314
|
+
missing = [h for h in declared if h not in headers]
|
|
315
|
+
if missing: raise SystemExit(f"Config names columns not in the file: {missing}. File has: {headers}")
|
|
316
|
+
return cfg
|
|
317
|
+
|
|
318
|
+
def list_sheets(path):
|
|
319
|
+
p = Path(path)
|
|
320
|
+
if p.suffix.lower() in (".xlsx", ".xlsm"):
|
|
321
|
+
wb = openpyxl.load_workbook(p, read_only=True, data_only=True); names = list(wb.sheetnames); wb.close(); return names
|
|
322
|
+
return []
|
|
323
|
+
|
|
324
|
+
def _read_grid(path, sheet):
|
|
325
|
+
p = Path(path); hint = None; sname = None; merges = []; fmiss = 0
|
|
326
|
+
if p.suffix.lower() in (".xlsx", ".xlsm"):
|
|
327
|
+
wb = openpyxl.load_workbook(p, data_only=True)
|
|
328
|
+
if sheet and sheet not in wb.sheetnames: raise SystemExit(f"Sheet '{sheet}' not found. Sheets: {wb.sheetnames}")
|
|
329
|
+
ws = wb[sheet] if sheet else wb.active; sname = ws.title
|
|
330
|
+
grid = [list(r) for r in ws.iter_rows(values_only=True)]
|
|
331
|
+
merges = [(m.min_row, m.min_col, m.max_row, m.max_col) for m in ws.merged_cells.ranges]
|
|
332
|
+
best = 0
|
|
333
|
+
for t in getattr(ws, "tables", {}).values(): # real Excel "Table" objects
|
|
334
|
+
try: c1, r1, c2, r2 = openpyxl.utils.cell.range_boundaries(t.ref)
|
|
335
|
+
except Exception: continue
|
|
336
|
+
if (r2 - r1 + 1) * (c2 - c1 + 1) > best: best = (r2 - r1 + 1) * (c2 - c1 + 1); hint = (r1, c1, r2, c2)
|
|
337
|
+
try: # formulas whose result was never saved read as empty
|
|
338
|
+
wsf = openpyxl.load_workbook(p, data_only=False)[ws.title]
|
|
339
|
+
for rf, rv in zip(wsf.iter_rows(), ws.iter_rows()):
|
|
340
|
+
for cf, cv in zip(rf, rv):
|
|
341
|
+
if isinstance(cf.value, str) and cf.value.startswith("=") and cv.value is None: fmiss += 1
|
|
342
|
+
except Exception: pass
|
|
343
|
+
elif p.suffix.lower() in (".csv", ".tsv", ".txt"):
|
|
344
|
+
raw = p.read_bytes()
|
|
345
|
+
try: text = raw.decode("utf-8-sig")
|
|
346
|
+
except UnicodeDecodeError: text = raw.decode("cp1252")
|
|
347
|
+
dialect = csv.excel_tab if p.suffix.lower() == ".tsv" else (csv.Sniffer().sniff(text[:4096], delimiters=",;\t") if text.strip() else csv.excel)
|
|
348
|
+
grid = [list(r) for r in csv.reader(text.splitlines(), dialect)]
|
|
349
|
+
else: raise SystemExit("Unsupported input type (use .xlsx or .csv)")
|
|
350
|
+
return grid, hint, sname, merges, fmiss
|
|
351
|
+
|
|
352
|
+
def _is_texty(c):
|
|
353
|
+
if isinstance(c, (datetime, date)): return False
|
|
354
|
+
v = norm_text(c)
|
|
355
|
+
return v != "" and not re.fullmatch(r"[\d\s.,:/\-+%$€]+", v)
|
|
356
|
+
|
|
357
|
+
def detect_header_row(grid):
|
|
358
|
+
"""1-based number of the (lowest) header row: skips title rows, blank rows and notes above the table."""
|
|
359
|
+
scan = grid[:60]; counts = [sum(1 for c in r if norm_text(c) != "") for r in scan]
|
|
360
|
+
maxn = max(counts) if counts else 0
|
|
361
|
+
if maxn == 0: raise SystemExit("The file/sheet is empty")
|
|
362
|
+
need = 1 if maxn == 1 else max(2, 0.6 * maxn)
|
|
363
|
+
for i, r in enumerate(scan):
|
|
364
|
+
n = counts[i]
|
|
365
|
+
if n < need: continue
|
|
366
|
+
if sum(1 for c in r if _is_texty(c)) < 0.6 * n: continue
|
|
367
|
+
nxt = counts[i + 1:i + 4]
|
|
368
|
+
if not nxt or any(x >= max(1, 0.5 * n) for x in nxt): return i + 1
|
|
369
|
+
return counts.index(maxn) + 1
|
|
370
|
+
|
|
371
|
+
TOTAL_RX = re.compile(ar_norm(r"^(total|totale|totaux|sous[- ]?total|grand total|subtotal|somme|sum|المجموع|مجموع|الاجمالي|اجمالي|الجملة|جملة)\b"), re.I)
|
|
372
|
+
|
|
373
|
+
def _is_total_row(vals):
|
|
374
|
+
for v in vals:
|
|
375
|
+
if norm_text(v) != "": return bool(TOTAL_RX.match(ar_norm(norm_text(v).casefold())))
|
|
376
|
+
return False
|
|
377
|
+
|
|
378
|
+
def read_table_info(path, sheet=None, header_row=None, levels=None, skip_totals=True):
|
|
379
|
+
"""Find the table. header_row = the LOWEST header row (the one with the sub-headings). levels=None detects group rows above it."""
|
|
380
|
+
grid, hint, sname, merges, fmiss = _read_grid(path, sheet)
|
|
381
|
+
if not any(norm_text(c) != "" for r in grid for c in r): raise SystemExit("The file/sheet is empty")
|
|
382
|
+
lo = hi = r_hi = None; how = "manual" if header_row else "auto"
|
|
383
|
+
if header_row is None:
|
|
384
|
+
if hint: header_row, lo, hi, r_hi = hint[0], hint[1] - 1, hint[3] - 1, hint[2]; how = "Excel table"
|
|
385
|
+
else: header_row = detect_header_row(grid)
|
|
386
|
+
if len(grid) < header_row: raise SystemExit("No header row found")
|
|
387
|
+
width = max(len(r) for r in grid); grid = [list(r) + [None] * (width - len(r)) for r in grid]
|
|
388
|
+
def filled(ri): # merged header cells repeat their label across the merge
|
|
389
|
+
row = list(grid[ri])
|
|
390
|
+
for r1, c1, r2, c2, in merges:
|
|
391
|
+
if r1 - 1 <= ri <= r2 - 1 and norm_text(grid[r1 - 1][c1 - 1]) != "":
|
|
392
|
+
for c in range(c1 - 1, min(c2, width)):
|
|
393
|
+
if norm_text(row[c]) == "": row[c] = grid[r1 - 1][c1 - 1]
|
|
394
|
+
return row
|
|
395
|
+
hdr = filled(header_row - 1); data = grid[header_row:r_hi]
|
|
396
|
+
lo = 0 if lo is None else lo; hi = width - 1 if hi is None else hi
|
|
397
|
+
keep = [c for c in range(lo, hi + 1) if norm_text(hdr[c]) != "" or any(norm_text(r[c]) != "" for r in data)]
|
|
398
|
+
if not keep: raise SystemExit("No data found below the header row")
|
|
399
|
+
group_rows = []; want = 2 if levels is None else max(0, levels - 1); cand = header_row - 1
|
|
400
|
+
while cand >= 1 and len(group_rows) < want:
|
|
401
|
+
row = filled(cand - 1); labs = [(c, norm_text(row[c])) for c in keep if norm_text(row[c]) != ""]
|
|
402
|
+
ok = bool(labs) if levels is not None else (len({l.casefold() for _, l in labs}) >= 2 and all(_is_texty(row[c]) for c, _ in labs))
|
|
403
|
+
if not ok: break
|
|
404
|
+
group_rows.append(cand); cand -= 1
|
|
405
|
+
group_rows.reverse()
|
|
406
|
+
subs = {c: norm_text(hdr[c]) for c in keep}; glabels = []
|
|
407
|
+
for gr in group_rows: # group labels: merged range, else nearest label above a sub-heading
|
|
408
|
+
row = filled(gr - 1); lab = {c: norm_text(row[c]) for c in keep}
|
|
409
|
+
cands = [c for c in keep if lab[c] and subs[c]]; res = {}
|
|
410
|
+
for c in keep:
|
|
411
|
+
if lab[c]: res[c] = lab[c]
|
|
412
|
+
elif subs[c] and cands: res[c] = lab[min(cands, key=lambda k: (abs(k - c), k > c))]
|
|
413
|
+
else: res[c] = ""
|
|
414
|
+
glabels.append(res)
|
|
415
|
+
headers, dnames, seen = [], {}, Counter()
|
|
416
|
+
for c in keep:
|
|
417
|
+
parts = []
|
|
418
|
+
for x in [g[c] for g in glabels] + [subs[c]]:
|
|
419
|
+
if x and (not parts or parts[-1].casefold() != x.casefold()): parts.append(x)
|
|
420
|
+
h = " - ".join(parts) or f"Column {get_column_letter(c + 1)}"; seen[h] += 1
|
|
421
|
+
h = h if seen[h] == 1 else f"{h} ({seen[h]})"; headers.append(h)
|
|
422
|
+
dnames[h] = subs[c] or (parts[-1] if parts else h)
|
|
423
|
+
rows, skipped, blanks = [], [], 0
|
|
424
|
+
for off, r in enumerate(data, start=header_row + 1):
|
|
425
|
+
vals = [r[c] for c in keep]
|
|
426
|
+
if all(norm_text(x) == "" for x in vals): blanks += 1; continue
|
|
427
|
+
if skip_totals and _is_total_row(vals): skipped.append({"src": off, "raw": vals, "reason": "total row"}); continue
|
|
428
|
+
rows.append({"src": off, "raw": vals})
|
|
429
|
+
top = group_rows[0] if group_rows else header_row
|
|
430
|
+
note = (f"sheet '{sname}', " if sname else "") + (f"header at row {header_row}" if not group_rows else f"header at rows {top}-{header_row} ({len(group_rows) + 1} levels)") \
|
|
431
|
+
+ f" ({how}), columns {get_column_letter(keep[0] + 1)}-{get_column_letter(keep[-1] + 1)}"
|
|
432
|
+
warnings = []
|
|
433
|
+
if fmiss: warnings.append(f"{fmiss} formula cell(s) have no saved result and read as empty: open the file in Excel, save it, then try again")
|
|
434
|
+
if skipped: warnings.append(f"{len(skipped)} total row(s) skipped, not cleaned (see sheet 'Skipped rows'): source row(s) " + ", ".join(str(k['src']) for k in skipped[:8]))
|
|
435
|
+
return {"headers": headers, "detect_names": dnames, "rows": rows, "skipped": skipped, "blanks": blanks, "header_row": header_row,
|
|
436
|
+
"levels": len(group_rows) + 1, "sheet": sname, "note": note, "how": how, "warnings": warnings}
|
|
437
|
+
|
|
438
|
+
def read_table(path, sheet=None, header_row=None):
|
|
439
|
+
i = read_table_info(path, sheet, header_row); return i["headers"], i["rows"], i["blanks"]
|
|
440
|
+
|
|
441
|
+
def clean_table(headers, rows, cfg):
|
|
442
|
+
n = len(rows); cells = {}; log = []
|
|
443
|
+
for j, h in enumerate(headers):
|
|
444
|
+
t = cfg["columns"][h]; raw = [r["raw"][j] for r in rows]
|
|
445
|
+
if t == "skip": res = [(x, "ok", "") for x in raw]
|
|
446
|
+
elif t in ("name", "category", "text"):
|
|
447
|
+
res = clean_text_column(raw, cfg, h, t)
|
|
448
|
+
elif t == "id":
|
|
449
|
+
res = [((norm_text(x) or None), "missing" if norm_text(x) == "" else "ok", "ID missing" if norm_text(x) == "" else "") for x in raw]
|
|
450
|
+
res = [(int(v) if isinstance(v, str) and v.isdigit() and not isinstance(x, str) else (x if isinstance(x, (int, float)) else v), s, m) for (v, s, m), x in zip(res, raw)]
|
|
451
|
+
else: res = [PARSERS[t](x, cfg) for x in raw]
|
|
452
|
+
for i, (v, s, m) in enumerate(res):
|
|
453
|
+
cells[(i, j)] = (v, s, m)
|
|
454
|
+
o = raw[i]
|
|
455
|
+
if s == "ok" and not same(o, v) and not (o is None and v in (None, "")):
|
|
456
|
+
log.append({"row": rows[i]["src"], "column": h, "before": o, "after": v, "status": "auto-fixed", "note": ""})
|
|
457
|
+
elif s in ("verify", "fail", "missing"):
|
|
458
|
+
st = {"verify": "VERIFY", "fail": "UNRESOLVED", "missing": "MISSING"}[s]
|
|
459
|
+
log.append({"row": rows[i]["src"], "column": h, "before": o, "after": v, "status": st, "note": m})
|
|
460
|
+
# exact duplicates (all columns identical after cleaning) -> removed, first kept
|
|
461
|
+
keyrow, removed, keep = {}, [], []
|
|
462
|
+
for i in range(n):
|
|
463
|
+
k = tuple(repr(cells[(i, j)][0]) for j in range(len(headers)))
|
|
464
|
+
if k in keyrow: removed.append((i, keyrow[k]))
|
|
465
|
+
else: keyrow[k] = i; keep.append(i)
|
|
466
|
+
for i, first in removed:
|
|
467
|
+
log.append({"row": rows[i]["src"], "column": "(entire row)", "before": "duplicate", "after": "removed",
|
|
468
|
+
"status": "auto-fixed", "note": f"exact duplicate of source row {rows[first]['src']}"})
|
|
469
|
+
# reused IDs / possible duplicates with different IDs (flag only)
|
|
470
|
+
idcols = [j for j, h in enumerate(headers) if cfg["columns"][h] == "id"]
|
|
471
|
+
if idcols:
|
|
472
|
+
j0 = idcols[0]; byid = defaultdict(list)
|
|
473
|
+
for i in keep:
|
|
474
|
+
if cells[(i, j0)][0] not in (None, ""): byid[cells[(i, j0)][0]].append(i)
|
|
475
|
+
for ident, idx in byid.items():
|
|
476
|
+
if len(idx) > 1:
|
|
477
|
+
for i in idx[1:]:
|
|
478
|
+
log.append({"row": rows[i]["src"], "column": headers[j0], "before": ident, "after": ident, "status": "VERIFY",
|
|
479
|
+
"note": f"ID also used by source row {rows[idx[0]]['src']} with different content"})
|
|
480
|
+
others = [j for j, h in enumerate(headers) if cfg["columns"][h] in ("name", "category", "qty", "price", "date", "phone", "email")]
|
|
481
|
+
seen = {}
|
|
482
|
+
for i in keep:
|
|
483
|
+
k = tuple(repr(cells[(i, j)][0]) for j in others)
|
|
484
|
+
if sum(1 for j in others if cells[(i, j)][0] not in (None, "")) >= 3:
|
|
485
|
+
if k in seen:
|
|
486
|
+
log.append({"row": rows[i]["src"], "column": "(entire row)", "before": "", "after": "kept", "status": "VERIFY",
|
|
487
|
+
"note": f"same content as source row {rows[seen[k]]['src']} but different ID; possible duplicate, not removed"})
|
|
488
|
+
else: seen[k] = i
|
|
489
|
+
return cells, keep, removed, log
|
|
490
|
+
|
|
491
|
+
def totals(headers, cfg, cells, keep):
|
|
492
|
+
q = [j for j, h in enumerate(headers) if cfg["columns"][h] == "qty"]; p = [j for j, h in enumerate(headers) if cfg["columns"][h] == "price"]
|
|
493
|
+
if not (len(q) == 1 and len(p) == 1): return None
|
|
494
|
+
out = {}
|
|
495
|
+
for i in keep:
|
|
496
|
+
a, b = cells[(i, q[0])][0], cells[(i, p[0])][0]
|
|
497
|
+
out[i] = round(a * b, 2) if isinstance(a, (int, float)) and isinstance(b, (int, float)) and not isinstance(a, bool) else None
|
|
498
|
+
return out
|
|
499
|
+
|
|
500
|
+
FILL_V = PatternFill("solid", fgColor="FFF2A8"); FILL_F = PatternFill("solid", fgColor="F8B88B")
|
|
501
|
+
HDR = PatternFill("solid", fgColor="1F3A5F")
|
|
502
|
+
|
|
503
|
+
def write_workbook(path, headers, rows, cfg, cells, keep, removed, log, blanks, src_name, table_note="", skipped=None, warnings=None):
|
|
504
|
+
wb = openpyxl.Workbook(); tot = totals(headers, cfg, cells, keep)
|
|
505
|
+
c = Counter(l["status"] for l in log); review = [l for l in log if l["status"] != "auto-fixed"]
|
|
506
|
+
ws = wb.active; ws.title = "Summary"
|
|
507
|
+
lines = [("Cleaning summary", ""), ("Source file", src_name), ("Table found", table_note), ("Rows read", len(rows) + 0), ("Blank rows skipped", blanks),
|
|
508
|
+
("Exact duplicate rows removed", len(removed)), ("Rows in cleaned output", len(keep)),
|
|
509
|
+
("Cells auto-fixed", c["auto-fixed"] - len(removed)), ("Cells to VERIFY (assumption made)", c["VERIFY"]),
|
|
510
|
+
("Cells UNRESOLVED (original kept)", c["UNRESOLVED"]), ("Cells MISSING", c["MISSING"]), ("", ""),
|
|
511
|
+
("Column types used", "")] + [(h, cfg["columns"][h] + (" (auto-detected)" if h in cfg["_auto_detected"] else "")) for h in headers]
|
|
512
|
+
for w_ in (warnings or []): lines += [("Note", w_)]
|
|
513
|
+
lines += [("", ""), ("Legend", "Yellow = changed with an assumption, please verify. Orange = could not be cleaned or missing; original kept.")]
|
|
514
|
+
for r in lines: ws.append(list(r))
|
|
515
|
+
ws["A1"].font = Font(bold=True, size=14); ws.column_dimensions["A"].width = 42; ws.column_dimensions["B"].width = 70
|
|
516
|
+
wc = wb.create_sheet("Clean"); outh = headers + (["Total (qty x price)"] if tot is not None else []) + ["Source row"]
|
|
517
|
+
wc.append(outh)
|
|
518
|
+
for k, i in enumerate(keep, start=2):
|
|
519
|
+
vals = [cells[(i, j)][0] for j in range(len(headers))]
|
|
520
|
+
if tot is not None: vals.append(tot[i])
|
|
521
|
+
wc.append(vals + [rows[i]["src"]])
|
|
522
|
+
for j in range(len(headers)):
|
|
523
|
+
v, s, _ = cells[(i, j)]; cell = wc.cell(row=k, column=j + 1)
|
|
524
|
+
if isinstance(v, date): cell.number_format = "yyyy-mm-dd"
|
|
525
|
+
if s == "verify": cell.fill = FILL_V
|
|
526
|
+
elif s in ("fail", "missing"): cell.fill = FILL_F
|
|
527
|
+
wr = wb.create_sheet("Review"); wr.append(["Source row", "Column", "Original", "Cleaned / kept", "Status", "What to check"])
|
|
528
|
+
for l in review: wr.append([l["row"], l["column"], str(l["before"]) if l["before"] is not None else "", str(l["after"]) if l["after"] is not None else "", l["status"], l["note"]])
|
|
529
|
+
wl = wb.create_sheet("Change log"); wl.append(["Source row", "Column", "Before", "After", "Status", "Note"])
|
|
530
|
+
for l in sorted(log, key=lambda x: (x["row"], x["column"])): wl.append([l["row"], l["column"], str(l["before"]) if l["before"] is not None else "", str(l["after"]) if l["after"] is not None else "", l["status"], l["note"]])
|
|
531
|
+
if skipped:
|
|
532
|
+
wsk = wb.create_sheet("Skipped rows"); wsk.append(["Source row", "Reason"] + headers)
|
|
533
|
+
for k in skipped: wsk.append([k["src"], k["reason"]] + [o.date() if isinstance(o, datetime) else o for o in k["raw"]])
|
|
534
|
+
wd = wb.create_sheet("Before-After (20 rows)"); h2 = ["Source row"]
|
|
535
|
+
for h in headers: h2 += [f"{h} (before)", f"{h} (after)"]
|
|
536
|
+
wd.append(h2)
|
|
537
|
+
for i in keep[:20]:
|
|
538
|
+
row = [rows[i]["src"]]
|
|
539
|
+
for j in range(len(headers)):
|
|
540
|
+
o = rows[i]["raw"][j]; v = cells[(i, j)][0]
|
|
541
|
+
row += [o.date() if isinstance(o, datetime) else o, v]
|
|
542
|
+
wd.append(row)
|
|
543
|
+
for w in [x for x in (wc, wr, wl, wd, wb["Skipped rows"] if "Skipped rows" in wb.sheetnames else None) if x is not None]:
|
|
544
|
+
for cell in w[1]: cell.font = Font(bold=True, color="FFFFFF"); cell.fill = HDR; cell.alignment = Alignment(wrap_text=True, vertical="top")
|
|
545
|
+
w.freeze_panes = "A2"
|
|
546
|
+
for idx, col in enumerate(w.columns, start=1):
|
|
547
|
+
width = max((len(str(x.value)) if x.value is not None else 0) for x in list(col)[:60])
|
|
548
|
+
w.column_dimensions[get_column_letter(idx)].width = min(max(10, width + 2), 48)
|
|
549
|
+
wb.save(path)
|
|
550
|
+
return lines
|
|
551
|
+
|
|
552
|
+
def report_md(src_name, headers, cfg, rows, keep, removed, log, blanks, cells, table_note="", warnings=None):
|
|
553
|
+
c = Counter(l["status"] for l in log); fixed = c["auto-fixed"] - len(removed)
|
|
554
|
+
L = [f"# Cleaning report - {src_name}", "", f"- Table found: {table_note}",
|
|
555
|
+
f"- Rows read: {len(rows)} | exact duplicates removed: {len(removed)} | rows in cleaned file: {len(keep)}",
|
|
556
|
+
f"- Auto-fixed cells: {fixed} | to VERIFY: {c['VERIFY']} | UNRESOLVED: {c['UNRESOLVED']} | MISSING: {c['MISSING']}", "",
|
|
557
|
+
*[f"- NOTE: {w_}" for w_ in (warnings or [])], "", "## What was changed automatically", ""]
|
|
558
|
+
per = Counter(l["column"] for l in log if l["status"] == "auto-fixed" and l["column"] != "(entire row)")
|
|
559
|
+
L += [f"- {h}: {n} cells (whitespace, casing, format)" for h, n in per.items()] or ["- none"]
|
|
560
|
+
L += ["", "## Needs a human decision", ""]
|
|
561
|
+
rev = [l for l in log if l["status"] != "auto-fixed"]
|
|
562
|
+
groups = defaultdict(list)
|
|
563
|
+
for l in rev:
|
|
564
|
+
shown = "(blank)" if l["before"] in (None, "") else str(l["before"])
|
|
565
|
+
note = re.sub(r"source row \d+", "source row N", l["note"]) if l["column"] == "(entire row)" or "also used" in l["note"] else l["note"]
|
|
566
|
+
groups[(l["column"], l["status"], shown if l["status"] != "MISSING" else "(blank)", note)].append(l["row"])
|
|
567
|
+
for (col, st, shown, note), rws in sorted(groups.items(), key=lambda kv: -len(kv[1]))[:20]:
|
|
568
|
+
rs = ", ".join(map(str, rws[:8])) + (f" +{len(rws) - 8} more" if len(rws) > 8 else "")
|
|
569
|
+
L.append(f"- {col} `{shown}` x{len(rws)} -> {st}: {note} (rows {rs})")
|
|
570
|
+
if not rev: L.append("- nothing flagged")
|
|
571
|
+
if len(groups) > 20: L.append(f"- ... {len(groups) - 20} more groups (see Review sheet)")
|
|
572
|
+
L += ["", "Nothing flagged above was silently guessed: yellow/orange cells in the Clean sheet mark each one. The original file was not modified."]
|
|
573
|
+
return "\n".join(L)
|
|
574
|
+
|
|
575
|
+
def run(path, cfg_path=None, out_dir=None, sheet=None, header_row=None, also_csv=False, write_config=None, levels=None, skip_totals=True, locale="dz"):
|
|
576
|
+
src = Path(path); info = read_table_info(src, sheet, header_row, levels, skip_totals); headers, rows, blanks = info["headers"], info["rows"], info["blanks"]
|
|
577
|
+
user_cfg = json.loads(Path(cfg_path).read_text(encoding="utf-8")) if cfg_path else {}
|
|
578
|
+
cfg = resolve_config(merge_locale(user_cfg, locale), headers, rows, info["detect_names"])
|
|
579
|
+
if write_config:
|
|
580
|
+
Path(write_config).write_text(json.dumps({k: v for k, v in cfg.items() if not k.startswith("_")}, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
581
|
+
out = Path(out_dir) if out_dir else src.parent; out.mkdir(parents=True, exist_ok=True)
|
|
582
|
+
target = out / f"{src.stem}_cleaned.xlsx"
|
|
583
|
+
if target.resolve() == src.resolve(): raise SystemExit("Refusing to overwrite the input file")
|
|
584
|
+
cells, keep, removed, log = clean_table(headers, rows, cfg)
|
|
585
|
+
write_workbook(target, headers, rows, cfg, cells, keep, removed, log, blanks, src.name, info["note"], info["skipped"], info["warnings"])
|
|
586
|
+
rep = out / f"{src.stem}_report.md"; rep.write_text(report_md(src.name, headers, cfg, rows, keep, removed, log, blanks, cells, info["note"], info["warnings"]), encoding="utf-8")
|
|
587
|
+
if also_csv:
|
|
588
|
+
with open(out / f"{src.stem}_cleaned.csv", "w", newline="", encoding="utf-8") as f:
|
|
589
|
+
w = csv.writer(f); w.writerow(headers)
|
|
590
|
+
for i in keep: w.writerow([("" if cells[(i, j)][0] is None else cells[(i, j)][0]) for j in range(len(headers))])
|
|
591
|
+
return {"xlsx": target, "report": rep, "rows_in": len(rows), "rows_out": len(keep), "removed": len(removed), "log": log, "cells": cells, "keep": keep, "headers": headers, "cfg": cfg, "info": info}
|
|
592
|
+
|
|
593
|
+
def main(argv=None):
|
|
594
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
595
|
+
ap.add_argument("input"); ap.add_argument("--config"); ap.add_argument("--out-dir"); ap.add_argument("--sheet")
|
|
596
|
+
ap.add_argument("--header-row", type=int, default=None, help="row number of the LOWEST header row"); ap.add_argument("--levels", type=int, default=None, help="header levels (1 = single header row); default: detect"); ap.add_argument("--keep-total-rows", action="store_true"); ap.add_argument("--also-csv", action="store_true"); ap.add_argument("--write-config")
|
|
597
|
+
ap.add_argument("--locale", default="dz", choices=sorted(LOCALES), help="country pack for phone/price/date rules (default: dz)")
|
|
598
|
+
a = ap.parse_args(argv)
|
|
599
|
+
r = run(a.input, a.config, a.out_dir, a.sheet, a.header_row, a.also_csv, a.write_config, a.levels, not a.keep_total_rows, a.locale)
|
|
600
|
+
c = Counter(l["status"] for l in r["log"])
|
|
601
|
+
print(f"{r['rows_in']} rows in -> {r['rows_out']} out ({r['removed']} duplicates removed). VERIFY={c['VERIFY']} UNRESOLVED={c['UNRESOLVED']} MISSING={c['MISSING']}")
|
|
602
|
+
print("Wrote:", r["xlsx"], "and", r["report"])
|
|
603
|
+
|
|
604
|
+
if __name__ == "__main__":
|
|
605
|
+
main()
|
clean_sheet_gui.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""CleanSheet - a simple window around clean_sheet.py (no command line needed).
|
|
3
|
+
Pick a file, check the detected table and column types, click Clean. Your original file is never modified."""
|
|
4
|
+
import json, os, subprocess, sys, traceback
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import tkinter as tk
|
|
8
|
+
from tkinter import filedialog, messagebox, ttk
|
|
9
|
+
|
|
10
|
+
import clean_sheet as cs
|
|
11
|
+
|
|
12
|
+
DATE_CHOICES = {"Day first (31/12/2026)": "dmy", "Month first (12/31/2026)": "mdy"}
|
|
13
|
+
TYPE_LABELS = { # friendly names shown in the window -> internal type
|
|
14
|
+
"Name": "name", "Category (city, product...)": "category", "Quantity": "qty", "Price / amount": "price", "Number (score, grade...)": "number",
|
|
15
|
+
"Date": "date", "Phone": "phone", "Email": "email", "ID / reference": "id", "Other text": "text", "Leave untouched": "skip"}
|
|
16
|
+
LABEL_OF = {v: k for k, v in TYPE_LABELS.items()}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def inspect_file(path, sheet=None, header_row=None, levels=None, skip_totals=True):
|
|
20
|
+
"""Find the table in the file. Returns headers, guessed types, example values, header row, sheets."""
|
|
21
|
+
info = cs.read_table_info(path, sheet, header_row, levels, skip_totals)
|
|
22
|
+
headers, rows = info["headers"], info["rows"]
|
|
23
|
+
cfg = cs.resolve_config({}, headers, rows, info["detect_names"])
|
|
24
|
+
samples = {}
|
|
25
|
+
for j, h in enumerate(headers):
|
|
26
|
+
vals = []
|
|
27
|
+
for r in rows:
|
|
28
|
+
v = cs.norm_text(r["raw"][j])
|
|
29
|
+
if v: vals.append(v[:28])
|
|
30
|
+
if len(vals) == 3: break
|
|
31
|
+
samples[h] = " | ".join(vals)
|
|
32
|
+
return {"headers": headers, "types": {h: cfg["columns"][h] for h in headers}, "samples": samples, "header_row": info["header_row"],
|
|
33
|
+
"how": info["how"], "sheet": info["sheet"], "sheets": cs.list_sheets(path), "note": info["note"], "rows": len(rows),
|
|
34
|
+
"levels": info["levels"], "warnings": info["warnings"], "skipped": len(info["skipped"])}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def run_job(path, columns, date_order="dmy", out_dir=None, also_csv=False, sheet=None, header_row=None, levels=None, skip_totals=True):
|
|
38
|
+
"""Run the cleaner with an explicit column mapping. Writes the config used next to the output."""
|
|
39
|
+
path = Path(path)
|
|
40
|
+
out = Path(out_dir) if out_dir else path.parent / "cleaned_output"
|
|
41
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
42
|
+
cfg_path = out / f"{path.stem}_used_config.json"
|
|
43
|
+
cfg_path.write_text(json.dumps({"columns": columns, "date_order": date_order}, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
44
|
+
r = cs.run(path, cfg_path, out, sheet, header_row, also_csv, None, levels, skip_totals)
|
|
45
|
+
c = Counter(l["status"] for l in r["log"])
|
|
46
|
+
return {"xlsx": r["xlsx"], "report": r["report"], "out": out, "rows_in": r["rows_in"], "rows_out": r["rows_out"],
|
|
47
|
+
"removed": r["removed"], "verify": c["VERIFY"], "unresolved": c["UNRESOLVED"], "missing": c["MISSING"], "note": r["info"]["note"], "skipped": len(r["info"]["skipped"]), "warnings": r["info"]["warnings"]}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def open_path(p):
|
|
51
|
+
p = str(p)
|
|
52
|
+
if sys.platform.startswith("win"): os.startfile(p)
|
|
53
|
+
elif sys.platform == "darwin": subprocess.Popen(["open", p])
|
|
54
|
+
else: subprocess.Popen(["xdg-open", p])
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class App(tk.Tk):
|
|
58
|
+
def __init__(self):
|
|
59
|
+
super().__init__()
|
|
60
|
+
self.title("CleanSheet - Excel/CSV cleaner"); self.geometry("900x720"); self.minsize(780, 620)
|
|
61
|
+
st = ttk.Style(self)
|
|
62
|
+
for theme in ("vista", "clam"):
|
|
63
|
+
if theme in st.theme_names(): st.theme_use(theme); break
|
|
64
|
+
base = ("Segoe UI", 10)
|
|
65
|
+
st.configure(".", font=base); st.configure("H.TLabel", font=("Segoe UI", 11, "bold")); st.configure("Hint.TLabel", foreground="#555")
|
|
66
|
+
st.configure("Big.TButton", font=("Segoe UI", 11, "bold"), padding=8)
|
|
67
|
+
self.path = None; self.sheet = None; self.header_override = None; self.levels_override = None; self.type_vars = {}; self.result = None
|
|
68
|
+
root = ttk.Frame(self, padding=14); root.pack(fill="both", expand=True)
|
|
69
|
+
|
|
70
|
+
ttk.Label(root, text="1. Choose your file", style="H.TLabel").grid(row=0, column=0, sticky="w")
|
|
71
|
+
top = ttk.Frame(root); top.grid(row=1, column=0, sticky="ew", pady=(4, 10)); top.columnconfigure(1, weight=1)
|
|
72
|
+
ttk.Button(top, text="Choose Excel / CSV file...", command=self.pick_file).grid(row=0, column=0)
|
|
73
|
+
self.file_lbl = ttk.Label(top, text="No file chosen", style="Hint.TLabel"); self.file_lbl.grid(row=0, column=1, sticky="w", padx=10)
|
|
74
|
+
self.table_row = ttk.Frame(top); self.table_row.grid(row=1, column=0, columnspan=2, sticky="w", pady=(8, 0))
|
|
75
|
+
ttk.Label(self.table_row, text="Sheet:").pack(side="left")
|
|
76
|
+
self.sheet_var = tk.StringVar(); self.sheet_cb = ttk.Combobox(self.table_row, textvariable=self.sheet_var, state="readonly", width=22)
|
|
77
|
+
self.sheet_cb.pack(side="left", padx=(4, 16)); self.sheet_cb.bind("<<ComboboxSelected>>", lambda e: self.reload(sheet_changed=True))
|
|
78
|
+
ttk.Label(self.table_row, text="Header row:").pack(side="left")
|
|
79
|
+
self.hdr_var = tk.StringVar(); self.hdr_sp = ttk.Spinbox(self.table_row, from_=1, to=500, width=5, textvariable=self.hdr_var, command=self.hdr_changed)
|
|
80
|
+
self.hdr_sp.pack(side="left", padx=4); self.hdr_sp.bind("<Return>", lambda e: self.hdr_changed()); self.hdr_sp.bind("<FocusOut>", lambda e: self.hdr_changed(only_if_changed=True))
|
|
81
|
+
ttk.Label(self.table_row, text=" Header levels:").pack(side="left")
|
|
82
|
+
self.lev_var = tk.StringVar(); self.lev_sp = ttk.Spinbox(self.table_row, from_=1, to=3, width=3, textvariable=self.lev_var, command=self.lev_changed)
|
|
83
|
+
self.lev_sp.pack(side="left", padx=4); self.lev_sp.bind("<Return>", lambda e: self.lev_changed())
|
|
84
|
+
self.skip_var = tk.BooleanVar(value=True); ttk.Checkbutton(self.table_row, text="Skip total rows", variable=self.skip_var, command=self.reload).pack(side="left", padx=10)
|
|
85
|
+
ttk.Button(self.table_row, text="Auto-detect again", command=self.auto_again).pack(side="left", padx=4)
|
|
86
|
+
self.detect_lbl = ttk.Label(root, text="", style="Hint.TLabel", wraplength=840, justify="left"); self.detect_lbl.grid(row=2, column=0, sticky="w")
|
|
87
|
+
|
|
88
|
+
ttk.Label(root, text="2. Check what each column contains", style="H.TLabel").grid(row=3, column=0, sticky="w", pady=(10, 0))
|
|
89
|
+
ttk.Label(root, text="The window guessed from the column names and the values. Change a type if it is wrong; 'Leave untouched' skips a column.", style="Hint.TLabel", wraplength=840, justify="left").grid(row=4, column=0, sticky="w")
|
|
90
|
+
box = ttk.Frame(root, relief="solid", borderwidth=1); box.grid(row=5, column=0, sticky="nsew", pady=6); root.rowconfigure(5, weight=1); root.columnconfigure(0, weight=1)
|
|
91
|
+
self.canvas = tk.Canvas(box, highlightthickness=0, background="white"); sb = ttk.Scrollbar(box, orient="vertical", command=self.canvas.yview)
|
|
92
|
+
self.cols_frame = ttk.Frame(self.canvas); self.cols_frame.bind("<Configure>", lambda e: self.canvas.configure(scrollregion=self.canvas.bbox("all")))
|
|
93
|
+
self.canvas.create_window((0, 0), window=self.cols_frame, anchor="nw"); self.canvas.configure(yscrollcommand=sb.set)
|
|
94
|
+
self.canvas.pack(side="left", fill="both", expand=True); sb.pack(side="right", fill="y")
|
|
95
|
+
self.canvas.bind_all("<MouseWheel>", lambda e: self.canvas.yview_scroll(int(-e.delta / 120), "units"))
|
|
96
|
+
|
|
97
|
+
opt = ttk.Frame(root); opt.grid(row=6, column=0, sticky="ew", pady=(4, 2))
|
|
98
|
+
ttk.Label(opt, text="Ambiguous dates like 03/04/2026:").pack(side="left")
|
|
99
|
+
self.date_var = tk.StringVar(value=list(DATE_CHOICES)[0])
|
|
100
|
+
ttk.Combobox(opt, textvariable=self.date_var, values=list(DATE_CHOICES), state="readonly", width=26).pack(side="left", padx=6)
|
|
101
|
+
self.csv_var = tk.BooleanVar(value=False); ttk.Checkbutton(opt, text="Also save a CSV copy", variable=self.csv_var).pack(side="left", padx=14)
|
|
102
|
+
out = ttk.Frame(root); out.grid(row=7, column=0, sticky="ew", pady=4); out.columnconfigure(1, weight=1)
|
|
103
|
+
ttk.Label(out, text="Save results in:").grid(row=0, column=0)
|
|
104
|
+
self.out_var = tk.StringVar(); ttk.Entry(out, textvariable=self.out_var).grid(row=0, column=1, sticky="ew", padx=6)
|
|
105
|
+
ttk.Button(out, text="Browse...", command=self.pick_out).grid(row=0, column=2)
|
|
106
|
+
|
|
107
|
+
ttk.Label(root, text="3. Clean", style="H.TLabel").grid(row=8, column=0, sticky="w", pady=(8, 2))
|
|
108
|
+
act = ttk.Frame(root); act.grid(row=9, column=0, sticky="w")
|
|
109
|
+
self.run_btn = ttk.Button(act, text="Clean file", style="Big.TButton", command=self.clean, state="disabled"); self.run_btn.pack(side="left")
|
|
110
|
+
self.open_file_btn = ttk.Button(act, text="Open cleaned file", command=lambda: open_path(self.result["xlsx"]), state="disabled"); self.open_file_btn.pack(side="left", padx=(14, 6))
|
|
111
|
+
self.open_dir_btn = ttk.Button(act, text="Open output folder", command=lambda: open_path(self.result["out"]), state="disabled"); self.open_dir_btn.pack(side="left")
|
|
112
|
+
self.result_lbl = ttk.Label(root, text="", wraplength=840, justify="left"); self.result_lbl.grid(row=10, column=0, sticky="w", pady=6)
|
|
113
|
+
ttk.Label(root, text="Your original file is never changed. In the 'Clean' sheet: yellow = changed with an assumption, orange = unreadable or missing. Check the 'Review' sheet by hand before sending anything.",
|
|
114
|
+
style="Hint.TLabel", wraplength=840, justify="left").grid(row=11, column=0, sticky="w")
|
|
115
|
+
self.table_row.grid_remove()
|
|
116
|
+
|
|
117
|
+
# ---- loading -------------------------------------------------------------------------
|
|
118
|
+
def pick_file(self):
|
|
119
|
+
p = filedialog.askopenfilename(title="Choose a file", filetypes=[("Excel or CSV", "*.xlsx *.xlsm *.csv *.tsv"), ("All files", "*.*")])
|
|
120
|
+
if p: self.load_file(p)
|
|
121
|
+
|
|
122
|
+
def load_file(self, p):
|
|
123
|
+
self.path = Path(p); self.sheet = None; self.header_override = None; self.levels_override = None
|
|
124
|
+
self.file_lbl.config(text=str(self.path)); self.out_var.set(str(self.path.parent / "cleaned_output"))
|
|
125
|
+
self.reload()
|
|
126
|
+
|
|
127
|
+
def reload(self, sheet_changed=False):
|
|
128
|
+
if not self.path: return
|
|
129
|
+
if sheet_changed: self.sheet = self.sheet_var.get() or None; self.header_override = None; self.levels_override = None
|
|
130
|
+
try: res = inspect_file(self.path, self.sheet, self.header_override, self.levels_override, self.skip_var.get())
|
|
131
|
+
except SystemExit as e: messagebox.showerror("Cannot read this table", str(e)); return
|
|
132
|
+
except Exception as e: messagebox.showerror("Cannot read file", f"{type(e).__name__}: {e}"); return
|
|
133
|
+
if res["sheets"]:
|
|
134
|
+
self.sheet_cb.config(values=res["sheets"]); self.sheet_var.set(res["sheet"]); self.sheet = res["sheet"]
|
|
135
|
+
self.table_row.grid()
|
|
136
|
+
self.hdr_var.set(str(res["header_row"])); self.last_hdr = str(res["header_row"]); self.lev_var.set(str(res["levels"]))
|
|
137
|
+
how = {"auto": "found automatically", "manual": "set by you", "Excel table": "taken from the Excel table in the file"}[res["how"]]
|
|
138
|
+
lev = "" if res["levels"] == 1 else f" The header has {res['levels']} levels (group title + sub-headings), shown joined like 'Group - Sub'."
|
|
139
|
+
extra = "".join("\n" + w for w in res["warnings"])
|
|
140
|
+
self.detect_lbl.config(text=f"Table: lowest header row {res['header_row']} ({how}); {res['rows']} data rows.{lev} If the names look wrong, change 'Header row' or 'Header levels' above.{extra}")
|
|
141
|
+
for w in self.cols_frame.winfo_children(): w.destroy()
|
|
142
|
+
for c, t in enumerate(("Column in your file", "What it contains", "Examples from your data")):
|
|
143
|
+
ttk.Label(self.cols_frame, text=t, font=("Segoe UI", 9, "bold")).grid(row=0, column=c, sticky="w", padx=8, pady=(6, 4))
|
|
144
|
+
self.type_vars = {}
|
|
145
|
+
for i, h in enumerate(res["headers"], start=1):
|
|
146
|
+
ttk.Label(self.cols_frame, text=h, width=28, anchor="w").grid(row=i, column=0, sticky="w", padx=8, pady=2)
|
|
147
|
+
v = tk.StringVar(value=LABEL_OF[res["types"][h]]); self.type_vars[h] = v
|
|
148
|
+
ttk.Combobox(self.cols_frame, textvariable=v, values=list(TYPE_LABELS), state="readonly", width=24).grid(row=i, column=1, padx=8, pady=2)
|
|
149
|
+
ttk.Label(self.cols_frame, text=res["samples"][h], foreground="#555").grid(row=i, column=2, sticky="w", padx=8)
|
|
150
|
+
self.run_btn.config(state="normal"); self.result_lbl.config(text="")
|
|
151
|
+
self.open_file_btn.config(state="disabled"); self.open_dir_btn.config(state="disabled")
|
|
152
|
+
|
|
153
|
+
def hdr_changed(self, only_if_changed=False):
|
|
154
|
+
v = self.hdr_var.get().strip()
|
|
155
|
+
if only_if_changed and v == getattr(self, "last_hdr", v): return
|
|
156
|
+
if not v.isdigit() or int(v) < 1: messagebox.showerror("Header row", "Enter a row number (1, 2, 3...)."); return
|
|
157
|
+
self.header_override = int(v); self.reload()
|
|
158
|
+
|
|
159
|
+
def lev_changed(self):
|
|
160
|
+
v = self.lev_var.get().strip()
|
|
161
|
+
if not v.isdigit() or not 1 <= int(v) <= 3: messagebox.showerror("Header levels", "Enter 1, 2 or 3."); return
|
|
162
|
+
self.levels_override = int(v); self.reload()
|
|
163
|
+
|
|
164
|
+
def auto_again(self):
|
|
165
|
+
self.header_override = None; self.levels_override = None; self.reload()
|
|
166
|
+
|
|
167
|
+
def pick_out(self):
|
|
168
|
+
d = filedialog.askdirectory(title="Choose output folder")
|
|
169
|
+
if d: self.out_var.set(d)
|
|
170
|
+
|
|
171
|
+
# ---- running -------------------------------------------------------------------------
|
|
172
|
+
def clean(self):
|
|
173
|
+
if not self.path: return
|
|
174
|
+
try:
|
|
175
|
+
cols = {h: TYPE_LABELS[v.get()] for h, v in self.type_vars.items()}
|
|
176
|
+
self.result = run_job(self.path, cols, DATE_CHOICES[self.date_var.get()], self.out_var.get() or None, self.csv_var.get(), self.sheet, self.header_override, self.levels_override, self.skip_var.get())
|
|
177
|
+
except SystemExit as e: messagebox.showerror("Cleaning stopped", str(e)); return
|
|
178
|
+
except Exception as e:
|
|
179
|
+
messagebox.showerror("Unexpected error", f"{type(e).__name__}: {e}\n\n{traceback.format_exc(limit=3)}"); return
|
|
180
|
+
r = self.result
|
|
181
|
+
self.result_lbl.config(text=f"Done: {r['rows_in']} rows in, {r['rows_out']} out ({r['removed']} exact duplicates removed).\n"
|
|
182
|
+
f"To check by hand: {r['verify']} yellow (assumption), {r['unresolved']} orange (unreadable), {r['missing']} missing."
|
|
183
|
+
+ (f" {r['skipped']} total row(s) were skipped (sheet 'Skipped rows').\n" if r["skipped"] else "\n") +
|
|
184
|
+
f"Saved in: {r['out']}")
|
|
185
|
+
self.open_file_btn.config(state="normal"); self.open_dir_btn.config(state="normal")
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def main():
|
|
189
|
+
App().mainloop()
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
if __name__ == "__main__":
|
|
193
|
+
main()
|
clean_sheet_locales.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Locale packs for clean_sheet.py.
|
|
3
|
+
|
|
4
|
+
A locale is a set of country-specific facts the cleaner needs: how phone
|
|
5
|
+
numbers are written, which currency words appear in price cells, and any
|
|
6
|
+
month names the global month table does not already know.
|
|
7
|
+
|
|
8
|
+
Each pack only sets defaults. Anything passed in a --config file wins.
|
|
9
|
+
|
|
10
|
+
The default locale is "dz" (Algeria): that is where the project started.
|
|
11
|
+
Use --locale generic for files that follow no single country's conventions.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
LOCALES = {
|
|
15
|
+
# Algeria: +213, 9 national digits (mobiles 05/06/07), DA/دج prices,
|
|
16
|
+
# Maghrebi French-Arabic month names (janvier, جانفي, جويلية...).
|
|
17
|
+
"dz": {
|
|
18
|
+
"country_code": "213",
|
|
19
|
+
"national_length": 9,
|
|
20
|
+
"mobile_prefixes": "23456789",
|
|
21
|
+
"leading_zero_required": True,
|
|
22
|
+
"date_order": "dmy",
|
|
23
|
+
"currency_tokens": ["dzd", "da", "dinars", "dinar", "eur", "usd", "€", "$",
|
|
24
|
+
"دينار جزائري", "دينار", "دنانير", "د.ج", "د ج", "دج"],
|
|
25
|
+
"months": {},
|
|
26
|
+
},
|
|
27
|
+
# Morocco: +212, 9 national digits (mobiles 06/07), MAD/درهم/DH prices,
|
|
28
|
+
# Moroccan month names (غشت for August, شتنبر for September, دجنبر for December).
|
|
29
|
+
"ma": {
|
|
30
|
+
"country_code": "212",
|
|
31
|
+
"national_length": 9,
|
|
32
|
+
"mobile_prefixes": "67",
|
|
33
|
+
"leading_zero_required": True,
|
|
34
|
+
"date_order": "dmy",
|
|
35
|
+
"currency_tokens": ["mad", "dhs", "dh", "dirhams", "dirham", "eur", "usd", "€", "$",
|
|
36
|
+
"الدرهم", "درهم", "د.م", "د م"],
|
|
37
|
+
"months": {"غشت": 8, "شتنبر": 9, "دجنبر": 12},
|
|
38
|
+
},
|
|
39
|
+
# Tunisia: +216, 8 national digits (fixed and mobile share one range),
|
|
40
|
+
# TND/DT/دينار prices. No mobile-prefix check: prefixes say nothing here.
|
|
41
|
+
"tn": {
|
|
42
|
+
"country_code": "216",
|
|
43
|
+
"national_length": 8,
|
|
44
|
+
"mobile_prefixes": "",
|
|
45
|
+
"leading_zero_required": False,
|
|
46
|
+
"date_order": "dmy",
|
|
47
|
+
"currency_tokens": ["tnd", "dt", "dinars", "dinar", "millimes", "millime", "eur", "usd", "€", "$",
|
|
48
|
+
"دينار", "دنانير", "د.ت", "د ت", "مليم"],
|
|
49
|
+
"months": {},
|
|
50
|
+
},
|
|
51
|
+
# Egypt: +20, 10 national digits (mobiles 010/011/012/015), EGP/جنيه prices,
|
|
52
|
+
# Eastern-Arabic month names (يناير, أغسطس...) already in the global table.
|
|
53
|
+
"eg": {
|
|
54
|
+
"country_code": "20",
|
|
55
|
+
"national_length": 10,
|
|
56
|
+
"mobile_prefixes": "101215",
|
|
57
|
+
"leading_zero_required": True,
|
|
58
|
+
"date_order": "dmy",
|
|
59
|
+
"currency_tokens": ["egp", "le", "جنيه", "ج.م", "ج م", "usd", "$"],
|
|
60
|
+
"months": {},
|
|
61
|
+
},
|
|
62
|
+
# France: +33, 9 national digits (mobiles 06/07), EUR prices, French months.
|
|
63
|
+
"fr": {
|
|
64
|
+
"country_code": "33",
|
|
65
|
+
"national_length": 9,
|
|
66
|
+
"mobile_prefixes": "67",
|
|
67
|
+
"leading_zero_required": True,
|
|
68
|
+
"date_order": "dmy",
|
|
69
|
+
"currency_tokens": ["eur", "€", "usd", "$", "euros", "euro"],
|
|
70
|
+
"months": {},
|
|
71
|
+
},
|
|
72
|
+
# Any other country: phones are only checked for a plausible length
|
|
73
|
+
# (8 to 15 digits), kept in their original form when international,
|
|
74
|
+
# and month names come from the shared global table
|
|
75
|
+
# (English, French, standard Arabic, Maghrebi and Levantine variants).
|
|
76
|
+
"generic": {
|
|
77
|
+
"country_code": "",
|
|
78
|
+
"national_length": None,
|
|
79
|
+
"mobile_prefixes": "",
|
|
80
|
+
"leading_zero_required": False,
|
|
81
|
+
"date_order": "dmy",
|
|
82
|
+
"currency_tokens": ["eur", "usd", "€", "$", "dinars", "dinar", "دينار"],
|
|
83
|
+
"months": {},
|
|
84
|
+
},
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
DESCRIPTIONS = {
|
|
88
|
+
"dz": "Algeria (+213, DA/دج, جانفي)",
|
|
89
|
+
"ma": "Morocco (+212, MAD/درهم, غشت)",
|
|
90
|
+
"tn": "Tunisia (+216, DT/د.ت, 8-digit numbers)",
|
|
91
|
+
"eg": "Egypt (+20, EGP/جنيه)",
|
|
92
|
+
"fr": "France (+33, €)",
|
|
93
|
+
"generic": "any other country (length checks only)",
|
|
94
|
+
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cleansheet
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Clean messy Excel/CSV files offline: logs every change and flags ambiguous values instead of guessing. Arabic, French and English headers, with locale packs for Algeria, Morocco, Tunisia, Egypt and France.
|
|
5
|
+
Author: aimenachraff-jpg
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/aimenachraff-jpg/CleanSheet
|
|
8
|
+
Project-URL: Repository, https://github.com/aimenachraff-jpg/CleanSheet
|
|
9
|
+
Project-URL: Issues, https://github.com/aimenachraff-jpg/CleanSheet/issues
|
|
10
|
+
Keywords: excel,csv,data-cleaning,arabic,openpyxl,algeria,morocco,tunisia,offline
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Environment :: Win32 (MS Windows)
|
|
14
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Natural Language :: Arabic
|
|
17
|
+
Classifier: Natural Language :: French
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
|
|
21
|
+
Requires-Python: >=3.8
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: openpyxl
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# CleanSheet
|
|
28
|
+
|
|
29
|
+
[](https://github.com/aimenachraff-jpg/CleanSheet/actions/workflows/tests.yml)
|
|
30
|
+
|
|
31
|
+
Cleans messy Excel and CSV files: extra spaces, inconsistent capitalization, exact duplicates, mixed price and date formats, numbers written as words, Arabic-Indic digits, country phone formats. You get the cleaned file, a log of every change, and a list of the values the tool could not decide on.
|
|
32
|
+
|
|
33
|
+
It is rule-based code, not AI. It never guesses: anything ambiguous is flagged in the output for you to decide. Your original file is never modified, and nothing leaves your computer.
|
|
34
|
+
|
|
35
|
+
Built with real sales data in mind: columns named in French, English or Arabic, files where the table does not start at A1, two-level headers, "Total" rows at the bottom, prices like `1 450 DA`, `1 250 DH` or `دج`, dates like `جانفي 2026` or `غشت 2026`.
|
|
36
|
+
|
|
37
|
+

|
|
38
|
+
|
|
39
|
+
<!-- Demo GIF (10-15s, messy file -> Clean -> yellow/orange review cells) goes here. -->
|
|
40
|
+
|
|
41
|
+
## Quick start
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
# Windows, no Python needed: download the exe from Releases, extract, run CleanSheet.exe
|
|
45
|
+
# Python, any OS:
|
|
46
|
+
pip install cleansheet
|
|
47
|
+
cleansheet file.xlsx # cleans file.xlsx, writes file_cleaned.xlsx
|
|
48
|
+
cleansheet file.xlsx --locale ma
|
|
49
|
+
cleansheet-gui # windowed app
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
More in [README.ar.md](README.ar.md) (العربية) and [README.fr.md](README.fr.md) (français).
|
|
53
|
+
|
|
54
|
+
## How it works
|
|
55
|
+
|
|
56
|
+
1. Open the app and pick a `.xlsx` or `.csv` file.
|
|
57
|
+
2. The tool finds the table (even below title rows or notes), guesses each column's type from its header, and shows you the guess to confirm or fix.
|
|
58
|
+
3. Click Clean. You get a new workbook in a `cleaned_output` folder next to your file, with five sheets:
|
|
59
|
+
|
|
60
|
+
| Sheet | What it holds |
|
|
61
|
+
|---|---|
|
|
62
|
+
| Clean | The cleaned data. Yellow = changed with an assumption, check it. Orange = unreadable or missing, original kept. |
|
|
63
|
+
| Review | Every yellow and orange cell with a note. Go through it before you use the file. |
|
|
64
|
+
| Change log | Every change, before and after. |
|
|
65
|
+
| Before-After | First 20 rows side by side. |
|
|
66
|
+
| Summary | Counts, plus the rules that were applied. |
|
|
67
|
+
|
|
68
|
+

|
|
69
|
+
|
|
70
|
+
Rows that look like totals ("Total", "Somme", "المجموع") are not cleaned as data; they are copied to a Skipped rows sheet. You can keep them in with one checkbox.
|
|
71
|
+
|
|
72
|
+
## Why not just use pandas, OpenRefine or Excel?
|
|
73
|
+
|
|
74
|
+
- **It runs offline and needs no skills.** A shop manager or accountant double-clicks the exe. pandas needs Python, a terminal and a script per file layout.
|
|
75
|
+
- **It never guesses.** pandas and Excel fix-converts silently: `02/09/2026` becomes September 2 without telling you. CleanSheet marks every assumption in yellow and leaves the call to you.
|
|
76
|
+
- **It knows the formats the big tools ignore.** `1 450 DA`, `1.250,50 درهم`, جانفي and غشت month names, `٠١٢٣` digits, 8-digit Tunisian phones. Out of the box, with a change log.
|
|
77
|
+
|
|
78
|
+
## What it fixes and what it does not
|
|
79
|
+
|
|
80
|
+
Fixed automatically (each one logged):
|
|
81
|
+
|
|
82
|
+
- Extra spaces, upper/lower case of the same name or city
|
|
83
|
+
- Number words like "dix" or "ثلاثة", price formats like `1 450 DA` or `1 250 DH`
|
|
84
|
+
- Clear dates like `3 septembre 2026` or `2026-09-03`; Arabic month names (Maghrebi, standard, Levantine) are understood
|
|
85
|
+
- Phone formats (+213, +212, +216, +20, +33), Arabic-Indic (٠١٢٣) and Persian (۰۱۲۳) digits
|
|
86
|
+
- Exact duplicate rows
|
|
87
|
+
- Typos in headers ("mongtant" → the montant column), two-level headers joined as `Produit1 - Qté`
|
|
88
|
+
|
|
89
|
+
Flagged for you, never decided for you:
|
|
90
|
+
|
|
91
|
+
- Ambiguous dates like `02/09/2026` (day-first or month-first; you pick the default, the tool marks them)
|
|
92
|
+
- Ambiguous numbers like `1,450` (1.45 or 1450)
|
|
93
|
+
- Unreadable or missing values: kept as they were, never filled in
|
|
94
|
+
- Names that differ only by hamza or diacritics (أحمد / احمد): flagged for review, never merged
|
|
95
|
+
- Synonyms across languages ("Thon" and "Tuna"): not merged, not flagged. Real synonyms need a human.
|
|
96
|
+
|
|
97
|
+
## Locale packs
|
|
98
|
+
|
|
99
|
+
Phone rules, currency words and month names come from a country pack, chosen with `--locale`:
|
|
100
|
+
|
|
101
|
+
| Locale | Country | Phones | Prices read |
|
|
102
|
+
|---|---|---|---|
|
|
103
|
+
| `dz` (default) | Algeria | +213, 9 digits | `1 450 DA`, `دج`, `دينار` |
|
|
104
|
+
| `ma` | Morocco | +212, 9 digits | `1 250 DH`, `درهم`, `MAD` |
|
|
105
|
+
| `tn` | Tunisia | +216, 8 digits, no leading 0 | `12,5 DT`, `د.ت` |
|
|
106
|
+
| `eg` | Egypt | +20, 10 digits | `1,250 LE`, `جنيه` |
|
|
107
|
+
| `fr` | France | +33, 9 digits | `1 250 €` |
|
|
108
|
+
| `generic` | anywhere | length check only | common currency symbols |
|
|
109
|
+
|
|
110
|
+
The default is `dz` (where the project started). See [CONTRIBUTING.md](CONTRIBUTING.md) to add a country; the pack file is a few lines. Sample files: `sample_messy_arabic.xlsx` (Arabic headers, Algerian formats), `sample_messy_moroccan.xlsx` (French headers, Moroccan formats), `sample_messy_sales.xlsx` (English).
|
|
111
|
+
|
|
112
|
+
## Getting it running
|
|
113
|
+
|
|
114
|
+
The easiest way, no Python needed: download [CleanSheet-1.0.0-windows.zip](https://github.com/aimenachraff-jpg/CleanSheet/releases/latest) from the Releases page, extract it, and run `CleanSheet.exe` from the extracted `CleanSheet` folder (Windows 10 or 11, 64-bit). Windows will show a SmartScreen warning because the exe is not signed; click More info, then Run anyway.
|
|
115
|
+
|
|
116
|
+
From Python:
|
|
117
|
+
|
|
118
|
+
```
|
|
119
|
+
pip install cleansheet
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Command line:
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
cleansheet client.xlsx --out-dir out/
|
|
126
|
+
# or from source: python clean_sheet.py client.xlsx
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Windowed app (pick the file, confirm the column types, click Clean):
|
|
130
|
+
|
|
131
|
+
```
|
|
132
|
+
cleansheet-gui
|
|
133
|
+
# or from source: python clean_sheet_gui.py
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
On Windows you can just double-click `start_CleanSheet.bat`.
|
|
137
|
+
|
|
138
|
+
### Building a standalone .exe
|
|
139
|
+
|
|
140
|
+
Double-click `build_exe.bat` (it downloads PyInstaller once). The result is `dist\CleanSheet\CleanSheet.exe`; keep the whole `CleanSheet` folder together and make a shortcut to the exe. Windows will show a SmartScreen warning for an unsigned exe you built yourself; that is expected.
|
|
141
|
+
|
|
142
|
+
### Column types
|
|
143
|
+
|
|
144
|
+
`clean_sheet.py` guesses column types from the header and, when the name gives no clue, from the values (phone, date, email, price with DA/دج/DH, plain numbers). To override the guess:
|
|
145
|
+
|
|
146
|
+
```
|
|
147
|
+
python clean_sheet.py client.xlsx --write-config client.json
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Edit `client.json` (types: `id`, `name`, `category`, `qty`, `price`, `date`, `phone`, `email`, `text`, `skip`), then rerun with `--config client.json`. Two example configs are in the repo: `config_sales_demo.json` and `config_client_style.json` (French headers). Other options: `--sheet NAME`, `--header-row N`, `--levels 1`, `--keep-total-rows`, `--also-csv`. A `--config` file always wins over the locale pack.
|
|
151
|
+
|
|
152
|
+
## Limitations
|
|
153
|
+
|
|
154
|
+
- Reads the first table on the chosen sheet only.
|
|
155
|
+
- No scans or PDFs, no formulas or dashboards, no merging files with different layouts.
|
|
156
|
+
- It does not invent values: what it cannot read stays unreadable, in orange.
|
|
157
|
+
- Tested on synthetic files. If it mishandles your file, please open an issue with a sample.
|
|
158
|
+
|
|
159
|
+
## Tests
|
|
160
|
+
|
|
161
|
+
```
|
|
162
|
+
python test_clean_sheet.py
|
|
163
|
+
python test_table_detection.py
|
|
164
|
+
python test_multilevel.py
|
|
165
|
+
python test_gui_logic.py
|
|
166
|
+
python test_locale_packs.py
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
57 tests cover the cleaning rules, table and header detection, the GUI logic, and the locale packs.
|
|
170
|
+
|
|
171
|
+
## License
|
|
172
|
+
|
|
173
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
clean_sheet.py,sha256=X_FY-lBFRcOv8qo8kS6xAeeLog9W9JJwZ7G3Mq40LA0,37883
|
|
2
|
+
clean_sheet_gui.py,sha256=OlfLjtMPm9R4JpyO4dpXVUZHSs4911Aj46kaBQZtkkk,14503
|
|
3
|
+
clean_sheet_locales.py,sha256=Cqob0nVkKW3ddoiKPWyVkuOtQbvg0-2aPnVevVte1Fg,3916
|
|
4
|
+
cleansheet-1.1.0.dist-info/licenses/LICENSE,sha256=9qvnOAQZcwCK4aymclC3MHNZzSjwC5OIIAmjulvN8YE,1073
|
|
5
|
+
cleansheet-1.1.0.dist-info/METADATA,sha256=rv38ydJqqoOKZwBcO2ovPYzZjbjHRpNyLh98mX3lcM0,8850
|
|
6
|
+
cleansheet-1.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
7
|
+
cleansheet-1.1.0.dist-info/entry_points.txt,sha256=OOuC0C94tA2BeCJ6_GkWRpmOD2R5cO7uNzKGa2xsXpg,86
|
|
8
|
+
cleansheet-1.1.0.dist-info/top_level.txt,sha256=CCYJz2piGspbENgDMbh1rhaLwsQspNlMwsEXMn1fNCA,48
|
|
9
|
+
cleansheet-1.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 aimenachraff-jpg
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|