csv2sef 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
csv2sef/pipeline.py ADDED
@@ -0,0 +1,485 @@
1
+ # -*- coding: utf-8 -*-
2
+ """Conversion pipeline: input table + metadata -> clean/ISO csv files and SEF files."""
3
+ import csv
4
+ import datetime as dt
5
+ import importlib.metadata
6
+ import os
7
+ from collections import Counter, defaultdict
8
+ from dataclasses import dataclass, field
9
+ from typing import Optional, Tuple
10
+
11
+ from . import metadata as mdmod
12
+ from . import reader, sef, tables
13
+ from .checks import cross_checks, range_check
14
+ from .convert import Context, Logger, process_value
15
+ from .headers import normalise, parse_time, parse_when, split_unit
16
+
17
+ # columns that identify the record rather than holding observations: ignored quietly
18
+ ID_COLUMNS = {"observer", "observer id", "observer_id", "station", "station id", "station_id",
19
+ "stationname", "station name", "timezone", "utcoffset", "utc offset", "utcdate",
20
+ "utc date", "fieldid", "field id", "field_id", "page", "page id", "page_id",
21
+ "image", "image file", "source", "id", "annotationid", "transcriptionid"}
22
+
23
+
24
+ @dataclass
25
+ class Column:
26
+ index: int
27
+ header: str
28
+ var: str
29
+ time: Optional[Tuple[int, int]]
30
+ time_label: Optional[str]
31
+ unit: str
32
+ unit_source: str
33
+ stat: str
34
+ period: str
35
+ vbl: str
36
+ quantity: str
37
+ description: str
38
+ header_meta: str = ""
39
+ suffix: str = ""
40
+ problems: list = field(default_factory=list)
41
+
42
+
43
+ # --------------------------------------------------------------------------
44
+ def identify_variable(text, md):
45
+ """'ta 07:00' -> ('ta', '07:00'); 'Thermometer Air AM' -> ('ta', 'AM'). None if unknown."""
46
+ known = tables.variables()
47
+ first, _, rest = text.partition(" ")
48
+ for cand in (first, first.strip("'\"")):
49
+ if cand in known or cand in md.variables:
50
+ return cand, rest
51
+ ci = [k for k in known if k.lower() == first.lower()]
52
+ if len(ci) == 1:
53
+ return ci[0], rest
54
+ name = tables.norm_name(text)
55
+ aliases = dict(tables.variable_aliases())
56
+ aliases.update({tables.norm_name(k): v for k, v in md.aliases.items()}) # user's first
57
+ for alias in sorted(aliases, key=len, reverse=True):
58
+ if name == alias or name.startswith(alias + " "):
59
+ # rest = remaining words of the original header
60
+ n_words = len(alias.split())
61
+ words = text.replace("_", " ").split()
62
+ return aliases[alias], " ".join(words[n_words:])
63
+ return None, None
64
+
65
+
66
+ def resolve_column(index, raw_header, md):
67
+ """Work out variable, time, unit and SEF settings for one data column."""
68
+ header = normalise(raw_header)
69
+ text, unit_token = split_unit(header)
70
+ var, rest = identify_variable(text, md)
71
+ if var is None:
72
+ return None, f"unknown variable '{text}' (see `csv2sef variables`; add alias.{text} = <code> " \
73
+ f"or ignore.{text} = yes to the metadata)"
74
+ known = tables.variables()
75
+ base = dict(known.get(var, {}))
76
+ over = dict(md.variables.get(var, {}))
77
+ for key in (header, text):
78
+ over.update(md.columns.get(key, {}))
79
+ if "variable" in over: # column re-assigned in the metadata
80
+ var = over.pop("variable")
81
+ base = dict(known.get(var, {}))
82
+ d = {**base, **over}
83
+ time, label, problem = parse_when(rest)
84
+ problems = [problem] if problem else []
85
+
86
+ # unit: header [unit] > metadata column/variable > unit.<var> > variables.csv default
87
+ default_unit = base.get("unit", "")
88
+ unit, source = None, "default"
89
+ for token, src in ((unit_token, "header"), (over.get("unit"), "metadata"),
90
+ (md.units.get(var), "metadata"), (default_unit, "default")):
91
+ if token and str(token).strip().lower() not in ("na", "n/a", "none", "-"):
92
+ u = tables.canonical_unit(token)
93
+ if u is None:
94
+ problems.append(f"unknown unit '{token}' ({src})")
95
+ continue
96
+ unit, source = u, src
97
+ break
98
+ if unit is None:
99
+ return None, "; ".join(problems) or f"no unit for '{var}'"
100
+ # a weather column marked [text] is still a weather description to be coded
101
+ if unit == "text" and default_unit == "mno":
102
+ unit, source = "mno", source + " (text -> weather codes)"
103
+
104
+ # time: metadata column/variable time= overrides the header
105
+ if "time" in over:
106
+ t = str(over["time"]).strip()
107
+ if t.lower() in ("", "none", "daily", "day"):
108
+ time, label = None, None
109
+ else:
110
+ time = parse_time(t)
111
+ label = None if time else t
112
+ if over.get("period") == "day" and "time" not in over:
113
+ time, label = None, None # column declared daily in the metadata
114
+ if time is None and label is None:
115
+ period = "day"
116
+ else:
117
+ period = over.get("period") or base.get("period", "0")
118
+ if label:
119
+ problems.append(f"approximate time '{label}': SEF Hour/Minute will be NA "
120
+ f"(set column.{header} = time=HH:MM in the metadata if known)")
121
+ return Column(index, header, var, time, label, unit, source,
122
+ d.get("stat", "point"), period, d.get("sef_vbl") or d.get("vbl") or var,
123
+ d.get("quantity", ""), d.get("description", ""),
124
+ d.get("header_meta", ""), d.get("suffix", ""), problems), None
125
+
126
+
127
+ def unit_on(md, var, date, unit):
128
+ for ch in md.unit_changes:
129
+ if ch["var"] != var:
130
+ continue
131
+ f = dt.date.fromisoformat(ch["from"]) if ch["from"] else dt.date.min
132
+ t = dt.date.fromisoformat(ch["to"]) if ch["to"] else dt.date.max
133
+ if f <= date <= t:
134
+ return tables.canonical_unit(ch["unit"]) or ch["unit"]
135
+ return unit
136
+
137
+
138
+ # --------------------------------------------------------------------------
139
+ def _empty(v):
140
+ return v.strip().lower() in ("", "na", "n/a", "nan", "none")
141
+
142
+
143
+ def observations(table, md, log):
144
+ """Return (list of (line, date, Column, raw value), columns, skipped, ignored, issues)."""
145
+ cols, julian = reader.date_columns(table.headers)
146
+ issues, columns, skipped, ignored = [], [], [], []
147
+ if not ("date" in cols or {"year", "month", "day"} <= set(cols)):
148
+ raise SystemExit("ERROR: no date columns found (need Year, Month, Day or Date)")
149
+
150
+ def is_ignored(h):
151
+ n = normalise(h).lower()
152
+ return n in ID_COLUMNS or n.replace(" ", "_") in ID_COLUMNS or n in md.ignore
153
+
154
+ if table.layout == "wide":
155
+ date_idx = set(cols.values()) | ({julian} if julian is not None else set())
156
+ for i, hd in enumerate(table.headers):
157
+ if i in date_idx or not hd:
158
+ continue
159
+ if is_ignored(hd):
160
+ ignored.append(hd)
161
+ continue
162
+ c, err = resolve_column(i, hd, md)
163
+ if c:
164
+ columns.append(c)
165
+ else:
166
+ skipped.append((hd, err))
167
+ else:
168
+ lc = reader.long_columns(table.headers)
169
+ used = set(lc.values()) | set(cols.values()) | ({julian} if julian is not None else set())
170
+ ignored = [h for i, h in enumerate(table.headers) if i not in used and h]
171
+ ignored_vars = []
172
+ cache = {}
173
+ midnight_daily = mdmod.truthy(md.get("midnight_means_daily", "no"))
174
+
175
+ seen = {}
176
+ out = []
177
+ for n, row in table.rows:
178
+ try:
179
+ date, dtime = reader.parse_datetime(row, cols)
180
+ except (ValueError, KeyError) as e:
181
+ log.write(Context(n, "date", None, ""), "DATE", f"Invalid date ({e}) - row skipped")
182
+ issues.append(f"line {n}: invalid date")
183
+ continue
184
+ if julian is not None and row[julian].strip():
185
+ try:
186
+ if int(float(row[julian])) != date.timetuple().tm_yday:
187
+ log.write(Context(n, "date", date, ""), "DATE",
188
+ f"Julian day {row[julian]} does not match date "
189
+ f"(day of year {date.timetuple().tm_yday})")
190
+ issues.append(f"line {n}: Julian day {row[julian]} does not match {date}")
191
+ except ValueError:
192
+ pass
193
+ if table.layout == "wide":
194
+ if date in seen:
195
+ log.write(Context(n, "date", date, ""), "DATE", f"Duplicate date (also line {seen[date]})")
196
+ issues.append(f"line {n}: duplicate date {date} (also line {seen[date]})")
197
+ seen[date] = n
198
+ for c in columns:
199
+ out.append((n, date, c, row[c.index]))
200
+ else:
201
+ hd = row[lc["variable"]].strip()
202
+ if is_ignored(hd):
203
+ if hd not in ignored:
204
+ ignored.append(hd)
205
+ continue
206
+ t = row[lc["time"]].strip() if "time" in lc else (dtime or "")
207
+ if t and not (midnight_daily and t in ("00:00", "0:00", "00:00:00")):
208
+ hd += " " + t
209
+ if "unit" in lc and row[lc["unit"]].strip():
210
+ hd += f" [{row[lc['unit']].strip()}]"
211
+ if hd not in cache:
212
+ c, err = resolve_column(-1, hd, md)
213
+ cache[hd] = c
214
+ if c:
215
+ columns.append(c)
216
+ else:
217
+ skipped.append((hd, err))
218
+ c = cache[hd]
219
+ if c:
220
+ val = row[lc["value"]]
221
+ key = (date, c.header)
222
+ if key in seen: # second value for the same date/time
223
+ j = seen[key]
224
+ prev = out[j][3]
225
+ if _empty(val):
226
+ continue # keep the earlier value
227
+ if _empty(prev):
228
+ out[j] = (n, date, c, val) # replace an empty earlier value
229
+ continue
230
+ if prev.strip() == val.strip():
231
+ continue # identical repeat
232
+ log.write(Context(n, c.header, date, c.var), "DUPLICATE",
233
+ f"Different value '{val}' for the same date/time "
234
+ f"(line {out[j][0]} has '{prev}'); both kept")
235
+ issues.append(f"line {n}: {c.header} on {date}: '{val}' vs '{prev}' (line {out[j][0]})")
236
+ else:
237
+ seen[key] = len(out)
238
+ out.append((n, date, c, val))
239
+ return out, columns, skipped, ignored, issues
240
+
241
+
242
+ # --------------------------------------------------------------------------
243
+ def build_records(obs, md, log, opts):
244
+ to_utc = mdmod.truthy(md.get("convert_to_utc", "yes"))
245
+ offset = md.site["utc_offset"]
246
+ daily_stamp = md.get("daily_timestamp", "next_midnight")
247
+ to_remarks = mdmod.truthy(md.get("unmapped_weather_to_remarks", "yes"))
248
+ records = []
249
+ for n, date, c, raw in obs:
250
+ unit = unit_on(md, c.var, date, c.unit)
251
+ if c.unit == "text" and _empty(raw):
252
+ continue # empty remark: nothing written
253
+ ctx = Context(n, c.header, date, c.var)
254
+ flag, clean, iso, iu = process_value(raw, c.var, unit, ctx, log, opts)
255
+ local = dt.datetime(date.year, date.month, date.day)
256
+ if c.time is not None:
257
+ local += dt.timedelta(hours=c.time[0], minutes=c.time[1])
258
+ stamp = local - dt.timedelta(hours=offset) if to_utc else local
259
+ hour, minute = stamp.hour, stamp.minute
260
+ elif c.time_label: # approximate time: Hour/Minute NA
261
+ stamp, hour, minute = local, None, None
262
+ elif daily_stamp == "next_midnight":
263
+ stamp = local + dt.timedelta(days=1)
264
+ hour, minute = 0, 0
265
+ else:
266
+ stamp, hour, minute = local, None, None
267
+ records.append({
268
+ "row": n, "date": date, "column": c.header, "var": c.var, "vbl": c.vbl,
269
+ "stat": c.stat, "period": c.period, "quantity": c.quantity, "suffix": c.suffix,
270
+ "header_meta": c.header_meta, "local_time": c.time, "time_label": c.time_label, "hour": c.time[0] if c.time else None,
271
+ "minute": c.time[1] if c.time else None, "stamp": stamp,
272
+ "sef_hour": hour, "sef_minute": minute,
273
+ "orig_value": raw.strip(), "orig_unit": unit, "clean_value": clean,
274
+ "iso_value": iso, "iso_unit": iu, "flag": flag,
275
+ })
276
+ if to_remarks and unit == "mno" and flag and "unmapped" in flag:
277
+ rec = dict(records[-1]) # weather text with no code -> remarks
278
+ rec.update({"var": "remarks", "vbl": "remarks", "stat": "point", "quantity": "",
279
+ "suffix": "", "header_meta": "", "orig_unit": "text", "clean_value": raw.strip(),
280
+ "iso_value": raw.strip(), "iso_unit": "text", "flag": None})
281
+ records[-1]["flag"] = flag + "; text in remarks"
282
+ records.append(rec)
283
+ return records
284
+
285
+
286
+ def options(md):
287
+ return {
288
+ "trace_mm": float(md.get("trace_mm", 0.2)),
289
+ "caves_temperature_c": md.get("caves_temperature_c", 12.8),
290
+ "below_scale_c": md.get("below_scale_c", ""),
291
+ "keep_wind_cloud_type": mdmod.truthy(md.get("keep_wind_cloud_type", "no")),
292
+ "wind_words": {**tables.wind_words(md.file_setting("wind_words_file")), **md.wind_words},
293
+ "weather_terms": md.weather_terms + tables.weather_terms(md.file_setting("weather_terms_file")),
294
+ }
295
+
296
+
297
+ # --------------------------------------------------------------------------
298
+ def run(data_path, meta_path, outdir=None, check_only=False, local_time=False, quiet=False):
299
+ md = mdmod.load(meta_path)
300
+ if local_time:
301
+ md.settings["convert_to_utc"] = "no"
302
+ site = md.site
303
+ table = reader.read(data_path, md.get("sheet"))
304
+ if outdir is None:
305
+ outdir = os.path.join(os.path.dirname(os.path.abspath(data_path)), f"sef_output_{site['id']}")
306
+ prefix = "_".join(p for p in (site.get("source", ""), site["id"]) if p)
307
+ if not check_only:
308
+ os.makedirs(os.path.join(outdir, "SEF"), exist_ok=True)
309
+ log = Logger(None if check_only else os.path.join(outdir, prefix + ".log"))
310
+ log.info(f"csv2sef {importlib.metadata.metadata(__package__)['Version']} input: {os.path.basename(data_path)} "
311
+ f"metadata: {os.path.basename(meta_path)}")
312
+ for note in md.notes:
313
+ log.info(note)
314
+
315
+ obs, columns, skipped, ignored, issues = observations(table, md, log)
316
+ opts = options(md)
317
+ records = build_records(obs, md, log, opts)
318
+ range_check(records, md.get("range_region", "global"), log)
319
+ cross_checks(records, log, mdmod.truthy(md.get("tn_night_after", "no")))
320
+
321
+ groups = group_records(records)
322
+ report = make_report(data_path, table, md, columns, skipped, ignored, issues, records, groups, log)
323
+ if check_only:
324
+ return report
325
+ write_csvs(records, site, outdir, prefix)
326
+ written = write_sef_files(groups, md, outdir)
327
+ log.info("Summary: " + ", ".join(f"{k}={v}" for k, v in sorted(log.counts.items())))
328
+ log.close()
329
+ report += "\n\nOutput: " + outdir + "\n" + "\n".join(
330
+ f" SEF/{f}: {n} lines, {ok} with values" for f, n, ok in written)
331
+ bad = {f: sef.validate(os.path.join(outdir, "SEF", f)) for f, _, _ in written}
332
+ bad = {f: p for f, p in bad.items() if p}
333
+ report += "\nSEF validation: " + ("all files valid" if not bad else
334
+ "; ".join(f"{f}: {p[0]}" for f, p in bad.items()))
335
+ return report
336
+
337
+
338
+ def group_records(records):
339
+ groups = defaultdict(list)
340
+ for r in records:
341
+ if r["iso_unit"] == "text" and r["var"] != "remarks":
342
+ continue
343
+ groups[(r["vbl"], r["stat"], r["period"], r["suffix"])].append(r)
344
+ count = Counter(k[0] for k in groups)
345
+ named = {}
346
+ for (vbl, stat, period, suffix), recs in groups.items():
347
+ if not suffix and count[vbl] > 1:
348
+ suffix = "_daily" if period == "day" else ("" if stat == "point" else f"_{stat}")
349
+ named[(vbl, stat, period, suffix)] = recs
350
+ return named
351
+
352
+
353
+ def write_csvs(records, site, outdir, prefix):
354
+ fields_clean = ["station", "year", "month", "day", "local_time", "column", "variable", "line",
355
+ "orig_value", "clean_value", "orig_unit", "flag"]
356
+ fields_iso = ["station", "year", "month", "day", "local_time", "column", "variable", "line",
357
+ "orig_value", "orig_unit", "iso_value", "iso_unit", "flag", "sef_datetime"]
358
+ with open(os.path.join(outdir, prefix + "_clean.csv"), "w", newline="", encoding="utf-8") as fc, \
359
+ open(os.path.join(outdir, prefix + "_iso.csv"), "w", newline="", encoding="utf-8") as fi:
360
+ wc, wi = csv.writer(fc), csv.writer(fi)
361
+ wc.writerow(fields_clean)
362
+ wi.writerow(fields_iso)
363
+ for r in records:
364
+ lt = (f"{r['local_time'][0]:02d}:{r['local_time'][1]:02d}" if r["local_time"]
365
+ else (r["time_label"] or "daily"))
366
+ base = [site["name"], r["date"].year, r["date"].month, r["date"].day, lt,
367
+ r["column"], r["var"], r["row"]]
368
+ wc.writerow(base + [r["orig_value"], "" if r["clean_value"] is None else r["clean_value"],
369
+ r["orig_unit"], r["flag"] or ""])
370
+ st = r["stamp"].strftime("%Y-%m-%d") + (
371
+ "" if r["sef_hour"] is None else f" {r['sef_hour']:02d}:{r['sef_minute']:02d}")
372
+ wi.writerow(base + [r["orig_value"], r["orig_unit"],
373
+ "" if r["iso_value"] is None else r["iso_value"],
374
+ r["iso_unit"], r["flag"] or "", st])
375
+
376
+
377
+ def write_sef_files(groups, md, outdir):
378
+ site = md.site
379
+ keep_na = mdmod.truthy(md.get("keep_na", "yes"))
380
+ to_utc = mdmod.truthy(md.get("convert_to_utc", "yes"))
381
+ missing = md.get("sef_missing_value", "NA")
382
+ written = []
383
+ for (vbl, stat, period, suffix), recs in sorted(groups.items()):
384
+ recs = sorted(recs, key=lambda r: (r["stamp"], r["column"]))
385
+ ok = [i for i, r in enumerate(recs) if r["iso_value"] is not None]
386
+ if not ok:
387
+ continue
388
+ recs = recs[ok[0]:ok[-1] + 1] # trim leading/trailing missing
389
+ if not keep_na:
390
+ recs = [r for r in recs if r["iso_value"] is not None]
391
+ hmeta = []
392
+ if site.get("data_policy"):
393
+ hmeta.append(f"Data policy={site['data_policy']}")
394
+ if site.get("observer"):
395
+ hmeta.append(f"Observer={site['observer']}")
396
+ if site.get("notes"):
397
+ hmeta.append(site["notes"])
398
+ if recs[0]["header_meta"]:
399
+ hmeta.append(recs[0]["header_meta"])
400
+ if period == "day":
401
+ hmeta.append("Daily values stamped 00:00 local time of the following day"
402
+ if md.get("daily_timestamp", "next_midnight") == "next_midnight" else
403
+ "Daily values: Hour and Minute not given")
404
+ else:
405
+ off = site["utc_offset"]
406
+ how = f" ({site['utc_offset_note']})" if site.get("utc_offset_note") else ""
407
+ hmeta.append(f"UTCOffset=YES|Local time offset={off:+g} h{how}" if to_utc
408
+ else f"UTCOffset=NO (local time; offset {off:+g} h{how})")
409
+ if any(r["orig_unit"] == "De_frz" for r in recs):
410
+ hmeta.append("orig De = Delisle degrees counted from freezing (C = De x 2/3)")
411
+ if any(r["time_label"] for r in recs):
412
+ hmeta.append("Hour and Minute NA where only an approximate time (time=...) is known")
413
+ hmeta.append(f"Converted with csv2sef {importlib.metadata.metadata(__package__)['Version']}")
414
+ rows = []
415
+ for r in recs:
416
+ meta = []
417
+ if r["orig_value"] and r["orig_unit"] not in ("text",):
418
+ ou = r["orig_unit"] if (r["orig_unit"] not in ("mno", "words", "dir", "lct", "uct")
419
+ and r["orig_value"][-1:].isdigit()) else ""
420
+ ou = {"De_frz": "De", "%": "%"}.get(ou, ou)
421
+ meta.append(f"orig={r['orig_value']}{ou}")
422
+ if r["flag"]:
423
+ meta.append(f"qc={r['flag']}")
424
+ if r["time_label"]:
425
+ meta.append(f"time={r['time_label']}")
426
+ rows.append({"year": r["stamp"].year, "month": r["stamp"].month, "day": r["stamp"].day,
427
+ "hour": r["sef_hour"], "minute": r["sef_minute"],
428
+ "period": period, "value": r["iso_value"], "meta": meta})
429
+ fname = sef.filename(site, vbl, recs[0]["date"], recs[-1]["date"], suffix)
430
+ sef.write(os.path.join(outdir, "SEF", fname), site, vbl, stat, recs[0]["iso_unit"],
431
+ "|".join(hmeta), rows, missing)
432
+ written.append((fname, len(rows), sum(1 for r in recs if r["iso_value"] is not None)))
433
+ return written
434
+
435
+
436
+ # --------------------------------------------------------------------------
437
+ def make_report(data_path, table, md, columns, skipped, ignored, issues, records, groups, log):
438
+ s = md.site
439
+ dates = sorted({r["date"] for r in records})
440
+ L = [f"csv2sef {importlib.metadata.metadata(__package__)['Version']} - {os.path.basename(data_path)}",
441
+ f"Station: {s['name']} (id {s['id']}) lat {s['lat']} lon {s['lon']} alt {s['alt']}",
442
+ f"Layout: {table.layout}; {len(table.rows)} data lines" +
443
+ (f"; {dates[0]} to {dates[-1]}" if dates else ""),
444
+ f"Times: {'converted to UTC' if mdmod.truthy(md.get('convert_to_utc', 'yes')) else 'local'}"
445
+ f" (local = UTC {s['utc_offset']:+g} h{', from longitude' if s.get('utc_offset_note') else ''})"]
446
+ L += [f"Note: {n}" for n in md.notes]
447
+ L.append("")
448
+ L.append(f"{'Column':28} {'Variable':9} {'Time':6} {'Unit':12} {'from':9} {'-> SI':8} {'Stat':8} "
449
+ f"{'values':>6} {'flagged':>7}")
450
+ by_col = defaultdict(list)
451
+ for r in records:
452
+ by_col[r["column"]].append(r)
453
+ for c in columns:
454
+ rs = by_col.get(c.header, [])
455
+ nval = sum(1 for r in rs if r["iso_value"] is not None)
456
+ nflag = sum(1 for r in rs if r["flag"] and r["flag"] != "missing")
457
+ t = (f"{c.time[0]:02d}:{c.time[1]:02d}" if c.time else
458
+ (f"~{c.time_label}"[:6] if c.time_label else "daily"))
459
+ iu = rs[0]["iso_unit"] if rs else ""
460
+ L.append(f"{c.header[:28]:28} {c.var:9} {t:6} {c.unit:12} {c.unit_source.split()[0]:9} "
461
+ f"{iu[:8]:8} {c.stat:8} {nval:6d} {nflag:7d}")
462
+ for p in c.problems:
463
+ L.append(f" ! {p}")
464
+ for hd, err in skipped:
465
+ L.append(f"{hd[:28]:28} SKIPPED: {err}")
466
+ if ignored:
467
+ L.append("Ignored (identifier columns or ignore.<name> in the metadata): " + ", ".join(ignored))
468
+ flags = Counter()
469
+ for r in records:
470
+ for f in (r["flag"] or "").split("; "):
471
+ if f:
472
+ flags[f] += 1
473
+ L.append("")
474
+ L.append("Flags: " + (", ".join(f"{k} {v}" for k, v in flags.most_common()) or "none"))
475
+ unm = Counter((r["column"], r["orig_value"]) for r in records
476
+ if r["flag"] and ("unmapped" in r["flag"] or "invalid" in r["flag"]))
477
+ if unm:
478
+ L.append("Values not converted (add them to wind_words.csv / weather_terms.csv or fix the data):")
479
+ for (col, v), k in unm.most_common(15):
480
+ L.append(f" {col}: '{v}' x{k}")
481
+ if issues:
482
+ L.append(f"Date problems ({len(issues)}):")
483
+ L += [" " + i for i in issues[:10]]
484
+ L.append("SEF files: " + ", ".join(f"{k[0]}{k[3]}" for k in sorted(groups)))
485
+ return "\n".join(L)
csv2sef/reader.py ADDED
@@ -0,0 +1,135 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Reading input files (csv, tsv, xlsx) in wide or long layout.
4
+
5
+ Wide layout (one row per day):
6
+ Year, Month, Day [, Julian day], <var time [unit]>, ...
7
+ or Date (YYYY-MM-DD), <var time [unit]>, ...
8
+ Long layout (one row per value): columns Date (or Year/Month/Day), Time (optional),
9
+ Variable, Value and optionally Unit.
10
+ """
11
+ import csv
12
+ import datetime as dt
13
+ import os
14
+ import re
15
+ from dataclasses import dataclass
16
+ from typing import Optional
17
+
18
+ from .headers import normalise
19
+
20
+ DATE_NAMES = {"year": "year", "yr": "year", "month": "month", "mo": "month", "day": "day",
21
+ "date": "date", "observationdate": "date", "observation date": "date",
22
+ "observation_date": "date", "obs_date": "date", "datetime": "date",
23
+ "date time": "date", "localdate": "date"}
24
+ JULIAN_WORDS = ("julian", "jday", "doy", "day of year")
25
+ # long layout column names
26
+ LONG_NAMES = {
27
+ "variable": {"variable", "field_key", "field key", "field", "element", "parameter", "vbl"},
28
+ "value": {"value", "obs_value", "observation", "val"},
29
+ "unit": {"unit", "units", "original unit"},
30
+ "time": {"time", "hour", "obs_time", "local time", "localtime"},
31
+ }
32
+
33
+
34
+ def long_columns(headers):
35
+ """name -> index for the long layout (variable, value, unit, time)."""
36
+ low = [h.lower() for h in headers]
37
+ out = {}
38
+ for key, names in LONG_NAMES.items():
39
+ for i, h in enumerate(low):
40
+ if h in names:
41
+ out[key] = i
42
+ break
43
+ return out
44
+
45
+
46
+ @dataclass
47
+ class Table:
48
+ headers: list
49
+ rows: list # list of (line_number, [cells as str])
50
+ layout: str # "wide" or "long"
51
+
52
+
53
+ def _cell(v):
54
+ if v is None:
55
+ return ""
56
+ if isinstance(v, (dt.datetime, dt.date)):
57
+ return v.strftime("%Y-%m-%d")
58
+ if isinstance(v, float):
59
+ return str(int(v)) if v.is_integer() else repr(v)
60
+ return str(v)
61
+
62
+
63
+ def unquote(v):
64
+ """Remove quotes wrapped around a whole cell: 'clear' -> clear."""
65
+ v = v.strip()
66
+ while len(v) >= 2 and v[0] == v[-1] and v[0] in "'\"":
67
+ v = v[1:-1].strip()
68
+ return v
69
+
70
+
71
+ def read(path, sheet=None):
72
+ ext = os.path.splitext(path)[1].lower()
73
+ if ext in (".xlsx", ".xlsm"):
74
+ try:
75
+ import openpyxl
76
+ except ImportError:
77
+ raise SystemExit("Reading Excel files needs openpyxl: pip install openpyxl")
78
+ wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
79
+ ws = wb[sheet] if sheet else wb.worksheets[0]
80
+ raw = [[_cell(c) for c in r] for r in ws.iter_rows(values_only=True)]
81
+ else:
82
+ raw = None
83
+ for enc in ("utf-8-sig", "cp1252"):
84
+ try:
85
+ with open(path, encoding=enc, newline="") as f:
86
+ text = f.read()
87
+ break
88
+ except UnicodeDecodeError:
89
+ continue
90
+ delim = "\t" if ext in (".tsv", ".tab") else ","
91
+ if ext not in (".tsv", ".tab"):
92
+ first = text.splitlines()[0] if text else ""
93
+ if first.count(";") > first.count(","):
94
+ delim = ";"
95
+ elif first.count("\t") > first.count(","):
96
+ delim = "\t"
97
+ raw = list(csv.reader(text.splitlines(), delimiter=delim))
98
+ # drop leading empty lines
99
+ while raw and not any(c.strip() for c in raw[0]):
100
+ raw.pop(0)
101
+ headers = [normalise(h) for h in raw[0]]
102
+ rows = [(i + 2, [unquote(c) for c in r] + [""] * (len(headers) - len(r)))
103
+ for i, r in enumerate(raw[1:]) if any(c.strip() for c in r)]
104
+ lc = long_columns(headers)
105
+ layout = "long" if ("variable" in lc and "value" in lc) else "wide"
106
+ return Table(headers, rows, layout)
107
+
108
+
109
+ def date_columns(headers):
110
+ """Return (dict name->index, julian index or None)."""
111
+ cols, julian = {}, None
112
+ for i, h in enumerate(headers):
113
+ hl = h.lower()
114
+ if hl in DATE_NAMES and DATE_NAMES[hl] not in cols:
115
+ cols[DATE_NAMES[hl]] = i
116
+ elif any(w in hl for w in JULIAN_WORDS) or hl == "jd" or (hl == "day" and "day" in cols):
117
+ julian = i
118
+ return cols, julian
119
+
120
+
121
+ def parse_date(row, cols):
122
+ """-> datetime.date or raises ValueError"""
123
+ return parse_datetime(row, cols)[0]
124
+
125
+
126
+ def parse_datetime(row, cols):
127
+ """-> (date, time string or None). A Date column may hold 'YYYY-MM-DD HH:MM[:SS]'."""
128
+ if "date" in cols:
129
+ s = row[cols["date"]].strip()
130
+ m = re.match(r"(\d{3,4})[-/.](\d{1,2})[-/.](\d{1,2})(?:[ T](\d{1,2}:\d{2})(?::\d{2})?)?\s*$", s)
131
+ if not m:
132
+ raise ValueError(f"date '{s}' not in YYYY-MM-DD form")
133
+ return dt.date(int(m.group(1)), int(m.group(2)), int(m.group(3))), m.group(4)
134
+ return dt.date(int(float(row[cols["year"]])), int(float(row[cols["month"]])),
135
+ int(float(row[cols["day"]]))), None