csv2sef 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csv2sef/__init__.py +5 -0
- csv2sef/checks.py +78 -0
- csv2sef/cli.py +126 -0
- csv2sef/convert.py +591 -0
- csv2sef/data/ranges.csv +17 -0
- csv2sef/data/units.csv +91 -0
- csv2sef/data/variable_aliases.csv +73 -0
- csv2sef/data/variables.csv +80 -0
- csv2sef/data/weather_terms.csv +71 -0
- csv2sef/data/wind_words.csv +40 -0
- csv2sef/headers.py +89 -0
- csv2sef/metadata.py +245 -0
- csv2sef/pipeline.py +485 -0
- csv2sef/reader.py +135 -0
- csv2sef/sef.py +116 -0
- csv2sef/tables.py +101 -0
- csv2sef-1.0.0.dist-info/METADATA +41 -0
- csv2sef-1.0.0.dist-info/RECORD +22 -0
- csv2sef-1.0.0.dist-info/WHEEL +5 -0
- csv2sef-1.0.0.dist-info/entry_points.txt +2 -0
- csv2sef-1.0.0.dist-info/licenses/LICENSE +21 -0
- csv2sef-1.0.0.dist-info/top_level.txt +1 -0
csv2sef/pipeline.py
ADDED
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Conversion pipeline: input table + metadata -> clean/ISO csv files and SEF files."""
|
|
3
|
+
import csv
|
|
4
|
+
import datetime as dt
|
|
5
|
+
import importlib.metadata
|
|
6
|
+
import os
|
|
7
|
+
from collections import Counter, defaultdict
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from typing import Optional, Tuple
|
|
10
|
+
|
|
11
|
+
from . import metadata as mdmod
|
|
12
|
+
from . import reader, sef, tables
|
|
13
|
+
from .checks import cross_checks, range_check
|
|
14
|
+
from .convert import Context, Logger, process_value
|
|
15
|
+
from .headers import normalise, parse_time, parse_when, split_unit
|
|
16
|
+
|
|
17
|
+
# columns that identify the record rather than holding observations: ignored quietly
|
|
18
|
+
ID_COLUMNS = {"observer", "observer id", "observer_id", "station", "station id", "station_id",
|
|
19
|
+
"stationname", "station name", "timezone", "utcoffset", "utc offset", "utcdate",
|
|
20
|
+
"utc date", "fieldid", "field id", "field_id", "page", "page id", "page_id",
|
|
21
|
+
"image", "image file", "source", "id", "annotationid", "transcriptionid"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class Column:
|
|
26
|
+
index: int
|
|
27
|
+
header: str
|
|
28
|
+
var: str
|
|
29
|
+
time: Optional[Tuple[int, int]]
|
|
30
|
+
time_label: Optional[str]
|
|
31
|
+
unit: str
|
|
32
|
+
unit_source: str
|
|
33
|
+
stat: str
|
|
34
|
+
period: str
|
|
35
|
+
vbl: str
|
|
36
|
+
quantity: str
|
|
37
|
+
description: str
|
|
38
|
+
header_meta: str = ""
|
|
39
|
+
suffix: str = ""
|
|
40
|
+
problems: list = field(default_factory=list)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# --------------------------------------------------------------------------
|
|
44
|
+
def identify_variable(text, md):
|
|
45
|
+
"""'ta 07:00' -> ('ta', '07:00'); 'Thermometer Air AM' -> ('ta', 'AM'). None if unknown."""
|
|
46
|
+
known = tables.variables()
|
|
47
|
+
first, _, rest = text.partition(" ")
|
|
48
|
+
for cand in (first, first.strip("'\"")):
|
|
49
|
+
if cand in known or cand in md.variables:
|
|
50
|
+
return cand, rest
|
|
51
|
+
ci = [k for k in known if k.lower() == first.lower()]
|
|
52
|
+
if len(ci) == 1:
|
|
53
|
+
return ci[0], rest
|
|
54
|
+
name = tables.norm_name(text)
|
|
55
|
+
aliases = dict(tables.variable_aliases())
|
|
56
|
+
aliases.update({tables.norm_name(k): v for k, v in md.aliases.items()}) # user's first
|
|
57
|
+
for alias in sorted(aliases, key=len, reverse=True):
|
|
58
|
+
if name == alias or name.startswith(alias + " "):
|
|
59
|
+
# rest = remaining words of the original header
|
|
60
|
+
n_words = len(alias.split())
|
|
61
|
+
words = text.replace("_", " ").split()
|
|
62
|
+
return aliases[alias], " ".join(words[n_words:])
|
|
63
|
+
return None, None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def resolve_column(index, raw_header, md):
|
|
67
|
+
"""Work out variable, time, unit and SEF settings for one data column."""
|
|
68
|
+
header = normalise(raw_header)
|
|
69
|
+
text, unit_token = split_unit(header)
|
|
70
|
+
var, rest = identify_variable(text, md)
|
|
71
|
+
if var is None:
|
|
72
|
+
return None, f"unknown variable '{text}' (see `csv2sef variables`; add alias.{text} = <code> " \
|
|
73
|
+
f"or ignore.{text} = yes to the metadata)"
|
|
74
|
+
known = tables.variables()
|
|
75
|
+
base = dict(known.get(var, {}))
|
|
76
|
+
over = dict(md.variables.get(var, {}))
|
|
77
|
+
for key in (header, text):
|
|
78
|
+
over.update(md.columns.get(key, {}))
|
|
79
|
+
if "variable" in over: # column re-assigned in the metadata
|
|
80
|
+
var = over.pop("variable")
|
|
81
|
+
base = dict(known.get(var, {}))
|
|
82
|
+
d = {**base, **over}
|
|
83
|
+
time, label, problem = parse_when(rest)
|
|
84
|
+
problems = [problem] if problem else []
|
|
85
|
+
|
|
86
|
+
# unit: header [unit] > metadata column/variable > unit.<var> > variables.csv default
|
|
87
|
+
default_unit = base.get("unit", "")
|
|
88
|
+
unit, source = None, "default"
|
|
89
|
+
for token, src in ((unit_token, "header"), (over.get("unit"), "metadata"),
|
|
90
|
+
(md.units.get(var), "metadata"), (default_unit, "default")):
|
|
91
|
+
if token and str(token).strip().lower() not in ("na", "n/a", "none", "-"):
|
|
92
|
+
u = tables.canonical_unit(token)
|
|
93
|
+
if u is None:
|
|
94
|
+
problems.append(f"unknown unit '{token}' ({src})")
|
|
95
|
+
continue
|
|
96
|
+
unit, source = u, src
|
|
97
|
+
break
|
|
98
|
+
if unit is None:
|
|
99
|
+
return None, "; ".join(problems) or f"no unit for '{var}'"
|
|
100
|
+
# a weather column marked [text] is still a weather description to be coded
|
|
101
|
+
if unit == "text" and default_unit == "mno":
|
|
102
|
+
unit, source = "mno", source + " (text -> weather codes)"
|
|
103
|
+
|
|
104
|
+
# time: metadata column/variable time= overrides the header
|
|
105
|
+
if "time" in over:
|
|
106
|
+
t = str(over["time"]).strip()
|
|
107
|
+
if t.lower() in ("", "none", "daily", "day"):
|
|
108
|
+
time, label = None, None
|
|
109
|
+
else:
|
|
110
|
+
time = parse_time(t)
|
|
111
|
+
label = None if time else t
|
|
112
|
+
if over.get("period") == "day" and "time" not in over:
|
|
113
|
+
time, label = None, None # column declared daily in the metadata
|
|
114
|
+
if time is None and label is None:
|
|
115
|
+
period = "day"
|
|
116
|
+
else:
|
|
117
|
+
period = over.get("period") or base.get("period", "0")
|
|
118
|
+
if label:
|
|
119
|
+
problems.append(f"approximate time '{label}': SEF Hour/Minute will be NA "
|
|
120
|
+
f"(set column.{header} = time=HH:MM in the metadata if known)")
|
|
121
|
+
return Column(index, header, var, time, label, unit, source,
|
|
122
|
+
d.get("stat", "point"), period, d.get("sef_vbl") or d.get("vbl") or var,
|
|
123
|
+
d.get("quantity", ""), d.get("description", ""),
|
|
124
|
+
d.get("header_meta", ""), d.get("suffix", ""), problems), None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def unit_on(md, var, date, unit):
|
|
128
|
+
for ch in md.unit_changes:
|
|
129
|
+
if ch["var"] != var:
|
|
130
|
+
continue
|
|
131
|
+
f = dt.date.fromisoformat(ch["from"]) if ch["from"] else dt.date.min
|
|
132
|
+
t = dt.date.fromisoformat(ch["to"]) if ch["to"] else dt.date.max
|
|
133
|
+
if f <= date <= t:
|
|
134
|
+
return tables.canonical_unit(ch["unit"]) or ch["unit"]
|
|
135
|
+
return unit
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# --------------------------------------------------------------------------
|
|
139
|
+
def _empty(v):
|
|
140
|
+
return v.strip().lower() in ("", "na", "n/a", "nan", "none")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def observations(table, md, log):
|
|
144
|
+
"""Return (list of (line, date, Column, raw value), columns, skipped, ignored, issues)."""
|
|
145
|
+
cols, julian = reader.date_columns(table.headers)
|
|
146
|
+
issues, columns, skipped, ignored = [], [], [], []
|
|
147
|
+
if not ("date" in cols or {"year", "month", "day"} <= set(cols)):
|
|
148
|
+
raise SystemExit("ERROR: no date columns found (need Year, Month, Day or Date)")
|
|
149
|
+
|
|
150
|
+
def is_ignored(h):
|
|
151
|
+
n = normalise(h).lower()
|
|
152
|
+
return n in ID_COLUMNS or n.replace(" ", "_") in ID_COLUMNS or n in md.ignore
|
|
153
|
+
|
|
154
|
+
if table.layout == "wide":
|
|
155
|
+
date_idx = set(cols.values()) | ({julian} if julian is not None else set())
|
|
156
|
+
for i, hd in enumerate(table.headers):
|
|
157
|
+
if i in date_idx or not hd:
|
|
158
|
+
continue
|
|
159
|
+
if is_ignored(hd):
|
|
160
|
+
ignored.append(hd)
|
|
161
|
+
continue
|
|
162
|
+
c, err = resolve_column(i, hd, md)
|
|
163
|
+
if c:
|
|
164
|
+
columns.append(c)
|
|
165
|
+
else:
|
|
166
|
+
skipped.append((hd, err))
|
|
167
|
+
else:
|
|
168
|
+
lc = reader.long_columns(table.headers)
|
|
169
|
+
used = set(lc.values()) | set(cols.values()) | ({julian} if julian is not None else set())
|
|
170
|
+
ignored = [h for i, h in enumerate(table.headers) if i not in used and h]
|
|
171
|
+
ignored_vars = []
|
|
172
|
+
cache = {}
|
|
173
|
+
midnight_daily = mdmod.truthy(md.get("midnight_means_daily", "no"))
|
|
174
|
+
|
|
175
|
+
seen = {}
|
|
176
|
+
out = []
|
|
177
|
+
for n, row in table.rows:
|
|
178
|
+
try:
|
|
179
|
+
date, dtime = reader.parse_datetime(row, cols)
|
|
180
|
+
except (ValueError, KeyError) as e:
|
|
181
|
+
log.write(Context(n, "date", None, ""), "DATE", f"Invalid date ({e}) - row skipped")
|
|
182
|
+
issues.append(f"line {n}: invalid date")
|
|
183
|
+
continue
|
|
184
|
+
if julian is not None and row[julian].strip():
|
|
185
|
+
try:
|
|
186
|
+
if int(float(row[julian])) != date.timetuple().tm_yday:
|
|
187
|
+
log.write(Context(n, "date", date, ""), "DATE",
|
|
188
|
+
f"Julian day {row[julian]} does not match date "
|
|
189
|
+
f"(day of year {date.timetuple().tm_yday})")
|
|
190
|
+
issues.append(f"line {n}: Julian day {row[julian]} does not match {date}")
|
|
191
|
+
except ValueError:
|
|
192
|
+
pass
|
|
193
|
+
if table.layout == "wide":
|
|
194
|
+
if date in seen:
|
|
195
|
+
log.write(Context(n, "date", date, ""), "DATE", f"Duplicate date (also line {seen[date]})")
|
|
196
|
+
issues.append(f"line {n}: duplicate date {date} (also line {seen[date]})")
|
|
197
|
+
seen[date] = n
|
|
198
|
+
for c in columns:
|
|
199
|
+
out.append((n, date, c, row[c.index]))
|
|
200
|
+
else:
|
|
201
|
+
hd = row[lc["variable"]].strip()
|
|
202
|
+
if is_ignored(hd):
|
|
203
|
+
if hd not in ignored:
|
|
204
|
+
ignored.append(hd)
|
|
205
|
+
continue
|
|
206
|
+
t = row[lc["time"]].strip() if "time" in lc else (dtime or "")
|
|
207
|
+
if t and not (midnight_daily and t in ("00:00", "0:00", "00:00:00")):
|
|
208
|
+
hd += " " + t
|
|
209
|
+
if "unit" in lc and row[lc["unit"]].strip():
|
|
210
|
+
hd += f" [{row[lc['unit']].strip()}]"
|
|
211
|
+
if hd not in cache:
|
|
212
|
+
c, err = resolve_column(-1, hd, md)
|
|
213
|
+
cache[hd] = c
|
|
214
|
+
if c:
|
|
215
|
+
columns.append(c)
|
|
216
|
+
else:
|
|
217
|
+
skipped.append((hd, err))
|
|
218
|
+
c = cache[hd]
|
|
219
|
+
if c:
|
|
220
|
+
val = row[lc["value"]]
|
|
221
|
+
key = (date, c.header)
|
|
222
|
+
if key in seen: # second value for the same date/time
|
|
223
|
+
j = seen[key]
|
|
224
|
+
prev = out[j][3]
|
|
225
|
+
if _empty(val):
|
|
226
|
+
continue # keep the earlier value
|
|
227
|
+
if _empty(prev):
|
|
228
|
+
out[j] = (n, date, c, val) # replace an empty earlier value
|
|
229
|
+
continue
|
|
230
|
+
if prev.strip() == val.strip():
|
|
231
|
+
continue # identical repeat
|
|
232
|
+
log.write(Context(n, c.header, date, c.var), "DUPLICATE",
|
|
233
|
+
f"Different value '{val}' for the same date/time "
|
|
234
|
+
f"(line {out[j][0]} has '{prev}'); both kept")
|
|
235
|
+
issues.append(f"line {n}: {c.header} on {date}: '{val}' vs '{prev}' (line {out[j][0]})")
|
|
236
|
+
else:
|
|
237
|
+
seen[key] = len(out)
|
|
238
|
+
out.append((n, date, c, val))
|
|
239
|
+
return out, columns, skipped, ignored, issues
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
# --------------------------------------------------------------------------
|
|
243
|
+
def build_records(obs, md, log, opts):
|
|
244
|
+
to_utc = mdmod.truthy(md.get("convert_to_utc", "yes"))
|
|
245
|
+
offset = md.site["utc_offset"]
|
|
246
|
+
daily_stamp = md.get("daily_timestamp", "next_midnight")
|
|
247
|
+
to_remarks = mdmod.truthy(md.get("unmapped_weather_to_remarks", "yes"))
|
|
248
|
+
records = []
|
|
249
|
+
for n, date, c, raw in obs:
|
|
250
|
+
unit = unit_on(md, c.var, date, c.unit)
|
|
251
|
+
if c.unit == "text" and _empty(raw):
|
|
252
|
+
continue # empty remark: nothing written
|
|
253
|
+
ctx = Context(n, c.header, date, c.var)
|
|
254
|
+
flag, clean, iso, iu = process_value(raw, c.var, unit, ctx, log, opts)
|
|
255
|
+
local = dt.datetime(date.year, date.month, date.day)
|
|
256
|
+
if c.time is not None:
|
|
257
|
+
local += dt.timedelta(hours=c.time[0], minutes=c.time[1])
|
|
258
|
+
stamp = local - dt.timedelta(hours=offset) if to_utc else local
|
|
259
|
+
hour, minute = stamp.hour, stamp.minute
|
|
260
|
+
elif c.time_label: # approximate time: Hour/Minute NA
|
|
261
|
+
stamp, hour, minute = local, None, None
|
|
262
|
+
elif daily_stamp == "next_midnight":
|
|
263
|
+
stamp = local + dt.timedelta(days=1)
|
|
264
|
+
hour, minute = 0, 0
|
|
265
|
+
else:
|
|
266
|
+
stamp, hour, minute = local, None, None
|
|
267
|
+
records.append({
|
|
268
|
+
"row": n, "date": date, "column": c.header, "var": c.var, "vbl": c.vbl,
|
|
269
|
+
"stat": c.stat, "period": c.period, "quantity": c.quantity, "suffix": c.suffix,
|
|
270
|
+
"header_meta": c.header_meta, "local_time": c.time, "time_label": c.time_label, "hour": c.time[0] if c.time else None,
|
|
271
|
+
"minute": c.time[1] if c.time else None, "stamp": stamp,
|
|
272
|
+
"sef_hour": hour, "sef_minute": minute,
|
|
273
|
+
"orig_value": raw.strip(), "orig_unit": unit, "clean_value": clean,
|
|
274
|
+
"iso_value": iso, "iso_unit": iu, "flag": flag,
|
|
275
|
+
})
|
|
276
|
+
if to_remarks and unit == "mno" and flag and "unmapped" in flag:
|
|
277
|
+
rec = dict(records[-1]) # weather text with no code -> remarks
|
|
278
|
+
rec.update({"var": "remarks", "vbl": "remarks", "stat": "point", "quantity": "",
|
|
279
|
+
"suffix": "", "header_meta": "", "orig_unit": "text", "clean_value": raw.strip(),
|
|
280
|
+
"iso_value": raw.strip(), "iso_unit": "text", "flag": None})
|
|
281
|
+
records[-1]["flag"] = flag + "; text in remarks"
|
|
282
|
+
records.append(rec)
|
|
283
|
+
return records
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def options(md):
|
|
287
|
+
return {
|
|
288
|
+
"trace_mm": float(md.get("trace_mm", 0.2)),
|
|
289
|
+
"caves_temperature_c": md.get("caves_temperature_c", 12.8),
|
|
290
|
+
"below_scale_c": md.get("below_scale_c", ""),
|
|
291
|
+
"keep_wind_cloud_type": mdmod.truthy(md.get("keep_wind_cloud_type", "no")),
|
|
292
|
+
"wind_words": {**tables.wind_words(md.file_setting("wind_words_file")), **md.wind_words},
|
|
293
|
+
"weather_terms": md.weather_terms + tables.weather_terms(md.file_setting("weather_terms_file")),
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
# --------------------------------------------------------------------------
|
|
298
|
+
def run(data_path, meta_path, outdir=None, check_only=False, local_time=False, quiet=False):
|
|
299
|
+
md = mdmod.load(meta_path)
|
|
300
|
+
if local_time:
|
|
301
|
+
md.settings["convert_to_utc"] = "no"
|
|
302
|
+
site = md.site
|
|
303
|
+
table = reader.read(data_path, md.get("sheet"))
|
|
304
|
+
if outdir is None:
|
|
305
|
+
outdir = os.path.join(os.path.dirname(os.path.abspath(data_path)), f"sef_output_{site['id']}")
|
|
306
|
+
prefix = "_".join(p for p in (site.get("source", ""), site["id"]) if p)
|
|
307
|
+
if not check_only:
|
|
308
|
+
os.makedirs(os.path.join(outdir, "SEF"), exist_ok=True)
|
|
309
|
+
log = Logger(None if check_only else os.path.join(outdir, prefix + ".log"))
|
|
310
|
+
log.info(f"csv2sef {importlib.metadata.metadata(__package__)['Version']} input: {os.path.basename(data_path)} "
|
|
311
|
+
f"metadata: {os.path.basename(meta_path)}")
|
|
312
|
+
for note in md.notes:
|
|
313
|
+
log.info(note)
|
|
314
|
+
|
|
315
|
+
obs, columns, skipped, ignored, issues = observations(table, md, log)
|
|
316
|
+
opts = options(md)
|
|
317
|
+
records = build_records(obs, md, log, opts)
|
|
318
|
+
range_check(records, md.get("range_region", "global"), log)
|
|
319
|
+
cross_checks(records, log, mdmod.truthy(md.get("tn_night_after", "no")))
|
|
320
|
+
|
|
321
|
+
groups = group_records(records)
|
|
322
|
+
report = make_report(data_path, table, md, columns, skipped, ignored, issues, records, groups, log)
|
|
323
|
+
if check_only:
|
|
324
|
+
return report
|
|
325
|
+
write_csvs(records, site, outdir, prefix)
|
|
326
|
+
written = write_sef_files(groups, md, outdir)
|
|
327
|
+
log.info("Summary: " + ", ".join(f"{k}={v}" for k, v in sorted(log.counts.items())))
|
|
328
|
+
log.close()
|
|
329
|
+
report += "\n\nOutput: " + outdir + "\n" + "\n".join(
|
|
330
|
+
f" SEF/{f}: {n} lines, {ok} with values" for f, n, ok in written)
|
|
331
|
+
bad = {f: sef.validate(os.path.join(outdir, "SEF", f)) for f, _, _ in written}
|
|
332
|
+
bad = {f: p for f, p in bad.items() if p}
|
|
333
|
+
report += "\nSEF validation: " + ("all files valid" if not bad else
|
|
334
|
+
"; ".join(f"{f}: {p[0]}" for f, p in bad.items()))
|
|
335
|
+
return report
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def group_records(records):
|
|
339
|
+
groups = defaultdict(list)
|
|
340
|
+
for r in records:
|
|
341
|
+
if r["iso_unit"] == "text" and r["var"] != "remarks":
|
|
342
|
+
continue
|
|
343
|
+
groups[(r["vbl"], r["stat"], r["period"], r["suffix"])].append(r)
|
|
344
|
+
count = Counter(k[0] for k in groups)
|
|
345
|
+
named = {}
|
|
346
|
+
for (vbl, stat, period, suffix), recs in groups.items():
|
|
347
|
+
if not suffix and count[vbl] > 1:
|
|
348
|
+
suffix = "_daily" if period == "day" else ("" if stat == "point" else f"_{stat}")
|
|
349
|
+
named[(vbl, stat, period, suffix)] = recs
|
|
350
|
+
return named
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def write_csvs(records, site, outdir, prefix):
|
|
354
|
+
fields_clean = ["station", "year", "month", "day", "local_time", "column", "variable", "line",
|
|
355
|
+
"orig_value", "clean_value", "orig_unit", "flag"]
|
|
356
|
+
fields_iso = ["station", "year", "month", "day", "local_time", "column", "variable", "line",
|
|
357
|
+
"orig_value", "orig_unit", "iso_value", "iso_unit", "flag", "sef_datetime"]
|
|
358
|
+
with open(os.path.join(outdir, prefix + "_clean.csv"), "w", newline="", encoding="utf-8") as fc, \
|
|
359
|
+
open(os.path.join(outdir, prefix + "_iso.csv"), "w", newline="", encoding="utf-8") as fi:
|
|
360
|
+
wc, wi = csv.writer(fc), csv.writer(fi)
|
|
361
|
+
wc.writerow(fields_clean)
|
|
362
|
+
wi.writerow(fields_iso)
|
|
363
|
+
for r in records:
|
|
364
|
+
lt = (f"{r['local_time'][0]:02d}:{r['local_time'][1]:02d}" if r["local_time"]
|
|
365
|
+
else (r["time_label"] or "daily"))
|
|
366
|
+
base = [site["name"], r["date"].year, r["date"].month, r["date"].day, lt,
|
|
367
|
+
r["column"], r["var"], r["row"]]
|
|
368
|
+
wc.writerow(base + [r["orig_value"], "" if r["clean_value"] is None else r["clean_value"],
|
|
369
|
+
r["orig_unit"], r["flag"] or ""])
|
|
370
|
+
st = r["stamp"].strftime("%Y-%m-%d") + (
|
|
371
|
+
"" if r["sef_hour"] is None else f" {r['sef_hour']:02d}:{r['sef_minute']:02d}")
|
|
372
|
+
wi.writerow(base + [r["orig_value"], r["orig_unit"],
|
|
373
|
+
"" if r["iso_value"] is None else r["iso_value"],
|
|
374
|
+
r["iso_unit"], r["flag"] or "", st])
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def write_sef_files(groups, md, outdir):
|
|
378
|
+
site = md.site
|
|
379
|
+
keep_na = mdmod.truthy(md.get("keep_na", "yes"))
|
|
380
|
+
to_utc = mdmod.truthy(md.get("convert_to_utc", "yes"))
|
|
381
|
+
missing = md.get("sef_missing_value", "NA")
|
|
382
|
+
written = []
|
|
383
|
+
for (vbl, stat, period, suffix), recs in sorted(groups.items()):
|
|
384
|
+
recs = sorted(recs, key=lambda r: (r["stamp"], r["column"]))
|
|
385
|
+
ok = [i for i, r in enumerate(recs) if r["iso_value"] is not None]
|
|
386
|
+
if not ok:
|
|
387
|
+
continue
|
|
388
|
+
recs = recs[ok[0]:ok[-1] + 1] # trim leading/trailing missing
|
|
389
|
+
if not keep_na:
|
|
390
|
+
recs = [r for r in recs if r["iso_value"] is not None]
|
|
391
|
+
hmeta = []
|
|
392
|
+
if site.get("data_policy"):
|
|
393
|
+
hmeta.append(f"Data policy={site['data_policy']}")
|
|
394
|
+
if site.get("observer"):
|
|
395
|
+
hmeta.append(f"Observer={site['observer']}")
|
|
396
|
+
if site.get("notes"):
|
|
397
|
+
hmeta.append(site["notes"])
|
|
398
|
+
if recs[0]["header_meta"]:
|
|
399
|
+
hmeta.append(recs[0]["header_meta"])
|
|
400
|
+
if period == "day":
|
|
401
|
+
hmeta.append("Daily values stamped 00:00 local time of the following day"
|
|
402
|
+
if md.get("daily_timestamp", "next_midnight") == "next_midnight" else
|
|
403
|
+
"Daily values: Hour and Minute not given")
|
|
404
|
+
else:
|
|
405
|
+
off = site["utc_offset"]
|
|
406
|
+
how = f" ({site['utc_offset_note']})" if site.get("utc_offset_note") else ""
|
|
407
|
+
hmeta.append(f"UTCOffset=YES|Local time offset={off:+g} h{how}" if to_utc
|
|
408
|
+
else f"UTCOffset=NO (local time; offset {off:+g} h{how})")
|
|
409
|
+
if any(r["orig_unit"] == "De_frz" for r in recs):
|
|
410
|
+
hmeta.append("orig De = Delisle degrees counted from freezing (C = De x 2/3)")
|
|
411
|
+
if any(r["time_label"] for r in recs):
|
|
412
|
+
hmeta.append("Hour and Minute NA where only an approximate time (time=...) is known")
|
|
413
|
+
hmeta.append(f"Converted with csv2sef {importlib.metadata.metadata(__package__)['Version']}")
|
|
414
|
+
rows = []
|
|
415
|
+
for r in recs:
|
|
416
|
+
meta = []
|
|
417
|
+
if r["orig_value"] and r["orig_unit"] not in ("text",):
|
|
418
|
+
ou = r["orig_unit"] if (r["orig_unit"] not in ("mno", "words", "dir", "lct", "uct")
|
|
419
|
+
and r["orig_value"][-1:].isdigit()) else ""
|
|
420
|
+
ou = {"De_frz": "De", "%": "%"}.get(ou, ou)
|
|
421
|
+
meta.append(f"orig={r['orig_value']}{ou}")
|
|
422
|
+
if r["flag"]:
|
|
423
|
+
meta.append(f"qc={r['flag']}")
|
|
424
|
+
if r["time_label"]:
|
|
425
|
+
meta.append(f"time={r['time_label']}")
|
|
426
|
+
rows.append({"year": r["stamp"].year, "month": r["stamp"].month, "day": r["stamp"].day,
|
|
427
|
+
"hour": r["sef_hour"], "minute": r["sef_minute"],
|
|
428
|
+
"period": period, "value": r["iso_value"], "meta": meta})
|
|
429
|
+
fname = sef.filename(site, vbl, recs[0]["date"], recs[-1]["date"], suffix)
|
|
430
|
+
sef.write(os.path.join(outdir, "SEF", fname), site, vbl, stat, recs[0]["iso_unit"],
|
|
431
|
+
"|".join(hmeta), rows, missing)
|
|
432
|
+
written.append((fname, len(rows), sum(1 for r in recs if r["iso_value"] is not None)))
|
|
433
|
+
return written
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
# --------------------------------------------------------------------------
|
|
437
|
+
def make_report(data_path, table, md, columns, skipped, ignored, issues, records, groups, log):
|
|
438
|
+
s = md.site
|
|
439
|
+
dates = sorted({r["date"] for r in records})
|
|
440
|
+
L = [f"csv2sef {importlib.metadata.metadata(__package__)['Version']} - {os.path.basename(data_path)}",
|
|
441
|
+
f"Station: {s['name']} (id {s['id']}) lat {s['lat']} lon {s['lon']} alt {s['alt']}",
|
|
442
|
+
f"Layout: {table.layout}; {len(table.rows)} data lines" +
|
|
443
|
+
(f"; {dates[0]} to {dates[-1]}" if dates else ""),
|
|
444
|
+
f"Times: {'converted to UTC' if mdmod.truthy(md.get('convert_to_utc', 'yes')) else 'local'}"
|
|
445
|
+
f" (local = UTC {s['utc_offset']:+g} h{', from longitude' if s.get('utc_offset_note') else ''})"]
|
|
446
|
+
L += [f"Note: {n}" for n in md.notes]
|
|
447
|
+
L.append("")
|
|
448
|
+
L.append(f"{'Column':28} {'Variable':9} {'Time':6} {'Unit':12} {'from':9} {'-> SI':8} {'Stat':8} "
|
|
449
|
+
f"{'values':>6} {'flagged':>7}")
|
|
450
|
+
by_col = defaultdict(list)
|
|
451
|
+
for r in records:
|
|
452
|
+
by_col[r["column"]].append(r)
|
|
453
|
+
for c in columns:
|
|
454
|
+
rs = by_col.get(c.header, [])
|
|
455
|
+
nval = sum(1 for r in rs if r["iso_value"] is not None)
|
|
456
|
+
nflag = sum(1 for r in rs if r["flag"] and r["flag"] != "missing")
|
|
457
|
+
t = (f"{c.time[0]:02d}:{c.time[1]:02d}" if c.time else
|
|
458
|
+
(f"~{c.time_label}"[:6] if c.time_label else "daily"))
|
|
459
|
+
iu = rs[0]["iso_unit"] if rs else ""
|
|
460
|
+
L.append(f"{c.header[:28]:28} {c.var:9} {t:6} {c.unit:12} {c.unit_source.split()[0]:9} "
|
|
461
|
+
f"{iu[:8]:8} {c.stat:8} {nval:6d} {nflag:7d}")
|
|
462
|
+
for p in c.problems:
|
|
463
|
+
L.append(f" ! {p}")
|
|
464
|
+
for hd, err in skipped:
|
|
465
|
+
L.append(f"{hd[:28]:28} SKIPPED: {err}")
|
|
466
|
+
if ignored:
|
|
467
|
+
L.append("Ignored (identifier columns or ignore.<name> in the metadata): " + ", ".join(ignored))
|
|
468
|
+
flags = Counter()
|
|
469
|
+
for r in records:
|
|
470
|
+
for f in (r["flag"] or "").split("; "):
|
|
471
|
+
if f:
|
|
472
|
+
flags[f] += 1
|
|
473
|
+
L.append("")
|
|
474
|
+
L.append("Flags: " + (", ".join(f"{k} {v}" for k, v in flags.most_common()) or "none"))
|
|
475
|
+
unm = Counter((r["column"], r["orig_value"]) for r in records
|
|
476
|
+
if r["flag"] and ("unmapped" in r["flag"] or "invalid" in r["flag"]))
|
|
477
|
+
if unm:
|
|
478
|
+
L.append("Values not converted (add them to wind_words.csv / weather_terms.csv or fix the data):")
|
|
479
|
+
for (col, v), k in unm.most_common(15):
|
|
480
|
+
L.append(f" {col}: '{v}' x{k}")
|
|
481
|
+
if issues:
|
|
482
|
+
L.append(f"Date problems ({len(issues)}):")
|
|
483
|
+
L += [" " + i for i in issues[:10]]
|
|
484
|
+
L.append("SEF files: " + ", ".join(f"{k[0]}{k[3]}" for k in sorted(groups)))
|
|
485
|
+
return "\n".join(L)
|
csv2sef/reader.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Reading input files (csv, tsv, xlsx) in wide or long layout.
|
|
4
|
+
|
|
5
|
+
Wide layout (one row per day):
|
|
6
|
+
Year, Month, Day [, Julian day], <var time [unit]>, ...
|
|
7
|
+
or Date (YYYY-MM-DD), <var time [unit]>, ...
|
|
8
|
+
Long layout (one row per value): columns Date (or Year/Month/Day), Time (optional),
|
|
9
|
+
Variable, Value and optionally Unit.
|
|
10
|
+
"""
|
|
11
|
+
import csv
|
|
12
|
+
import datetime as dt
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import Optional
|
|
17
|
+
|
|
18
|
+
from .headers import normalise
|
|
19
|
+
|
|
20
|
+
DATE_NAMES = {"year": "year", "yr": "year", "month": "month", "mo": "month", "day": "day",
|
|
21
|
+
"date": "date", "observationdate": "date", "observation date": "date",
|
|
22
|
+
"observation_date": "date", "obs_date": "date", "datetime": "date",
|
|
23
|
+
"date time": "date", "localdate": "date"}
|
|
24
|
+
JULIAN_WORDS = ("julian", "jday", "doy", "day of year")
|
|
25
|
+
# long layout column names
|
|
26
|
+
LONG_NAMES = {
|
|
27
|
+
"variable": {"variable", "field_key", "field key", "field", "element", "parameter", "vbl"},
|
|
28
|
+
"value": {"value", "obs_value", "observation", "val"},
|
|
29
|
+
"unit": {"unit", "units", "original unit"},
|
|
30
|
+
"time": {"time", "hour", "obs_time", "local time", "localtime"},
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def long_columns(headers):
|
|
35
|
+
"""name -> index for the long layout (variable, value, unit, time)."""
|
|
36
|
+
low = [h.lower() for h in headers]
|
|
37
|
+
out = {}
|
|
38
|
+
for key, names in LONG_NAMES.items():
|
|
39
|
+
for i, h in enumerate(low):
|
|
40
|
+
if h in names:
|
|
41
|
+
out[key] = i
|
|
42
|
+
break
|
|
43
|
+
return out
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class Table:
|
|
48
|
+
headers: list
|
|
49
|
+
rows: list # list of (line_number, [cells as str])
|
|
50
|
+
layout: str # "wide" or "long"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _cell(v):
|
|
54
|
+
if v is None:
|
|
55
|
+
return ""
|
|
56
|
+
if isinstance(v, (dt.datetime, dt.date)):
|
|
57
|
+
return v.strftime("%Y-%m-%d")
|
|
58
|
+
if isinstance(v, float):
|
|
59
|
+
return str(int(v)) if v.is_integer() else repr(v)
|
|
60
|
+
return str(v)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def unquote(v):
|
|
64
|
+
"""Remove quotes wrapped around a whole cell: 'clear' -> clear."""
|
|
65
|
+
v = v.strip()
|
|
66
|
+
while len(v) >= 2 and v[0] == v[-1] and v[0] in "'\"":
|
|
67
|
+
v = v[1:-1].strip()
|
|
68
|
+
return v
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def read(path, sheet=None):
|
|
72
|
+
ext = os.path.splitext(path)[1].lower()
|
|
73
|
+
if ext in (".xlsx", ".xlsm"):
|
|
74
|
+
try:
|
|
75
|
+
import openpyxl
|
|
76
|
+
except ImportError:
|
|
77
|
+
raise SystemExit("Reading Excel files needs openpyxl: pip install openpyxl")
|
|
78
|
+
wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
|
|
79
|
+
ws = wb[sheet] if sheet else wb.worksheets[0]
|
|
80
|
+
raw = [[_cell(c) for c in r] for r in ws.iter_rows(values_only=True)]
|
|
81
|
+
else:
|
|
82
|
+
raw = None
|
|
83
|
+
for enc in ("utf-8-sig", "cp1252"):
|
|
84
|
+
try:
|
|
85
|
+
with open(path, encoding=enc, newline="") as f:
|
|
86
|
+
text = f.read()
|
|
87
|
+
break
|
|
88
|
+
except UnicodeDecodeError:
|
|
89
|
+
continue
|
|
90
|
+
delim = "\t" if ext in (".tsv", ".tab") else ","
|
|
91
|
+
if ext not in (".tsv", ".tab"):
|
|
92
|
+
first = text.splitlines()[0] if text else ""
|
|
93
|
+
if first.count(";") > first.count(","):
|
|
94
|
+
delim = ";"
|
|
95
|
+
elif first.count("\t") > first.count(","):
|
|
96
|
+
delim = "\t"
|
|
97
|
+
raw = list(csv.reader(text.splitlines(), delimiter=delim))
|
|
98
|
+
# drop leading empty lines
|
|
99
|
+
while raw and not any(c.strip() for c in raw[0]):
|
|
100
|
+
raw.pop(0)
|
|
101
|
+
headers = [normalise(h) for h in raw[0]]
|
|
102
|
+
rows = [(i + 2, [unquote(c) for c in r] + [""] * (len(headers) - len(r)))
|
|
103
|
+
for i, r in enumerate(raw[1:]) if any(c.strip() for c in r)]
|
|
104
|
+
lc = long_columns(headers)
|
|
105
|
+
layout = "long" if ("variable" in lc and "value" in lc) else "wide"
|
|
106
|
+
return Table(headers, rows, layout)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def date_columns(headers):
|
|
110
|
+
"""Return (dict name->index, julian index or None)."""
|
|
111
|
+
cols, julian = {}, None
|
|
112
|
+
for i, h in enumerate(headers):
|
|
113
|
+
hl = h.lower()
|
|
114
|
+
if hl in DATE_NAMES and DATE_NAMES[hl] not in cols:
|
|
115
|
+
cols[DATE_NAMES[hl]] = i
|
|
116
|
+
elif any(w in hl for w in JULIAN_WORDS) or hl == "jd" or (hl == "day" and "day" in cols):
|
|
117
|
+
julian = i
|
|
118
|
+
return cols, julian
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def parse_date(row, cols):
|
|
122
|
+
"""-> datetime.date or raises ValueError"""
|
|
123
|
+
return parse_datetime(row, cols)[0]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def parse_datetime(row, cols):
|
|
127
|
+
"""-> (date, time string or None). A Date column may hold 'YYYY-MM-DD HH:MM[:SS]'."""
|
|
128
|
+
if "date" in cols:
|
|
129
|
+
s = row[cols["date"]].strip()
|
|
130
|
+
m = re.match(r"(\d{3,4})[-/.](\d{1,2})[-/.](\d{1,2})(?:[ T](\d{1,2}:\d{2})(?::\d{2})?)?\s*$", s)
|
|
131
|
+
if not m:
|
|
132
|
+
raise ValueError(f"date '{s}' not in YYYY-MM-DD form")
|
|
133
|
+
return dt.date(int(m.group(1)), int(m.group(2)), int(m.group(3))), m.group(4)
|
|
134
|
+
return dt.date(int(float(row[cols["year"]])), int(float(row[cols["month"]])),
|
|
135
|
+
int(float(row[cols["day"]]))), None
|