csv2sef 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csv2sef/__init__.py +5 -0
- csv2sef/checks.py +78 -0
- csv2sef/cli.py +126 -0
- csv2sef/convert.py +591 -0
- csv2sef/data/ranges.csv +17 -0
- csv2sef/data/units.csv +91 -0
- csv2sef/data/variable_aliases.csv +73 -0
- csv2sef/data/variables.csv +80 -0
- csv2sef/data/weather_terms.csv +71 -0
- csv2sef/data/wind_words.csv +40 -0
- csv2sef/headers.py +89 -0
- csv2sef/metadata.py +245 -0
- csv2sef/pipeline.py +485 -0
- csv2sef/reader.py +135 -0
- csv2sef/sef.py +116 -0
- csv2sef/tables.py +101 -0
- csv2sef-1.0.0.dist-info/METADATA +41 -0
- csv2sef-1.0.0.dist-info/RECORD +22 -0
- csv2sef-1.0.0.dist-info/WHEEL +5 -0
- csv2sef-1.0.0.dist-info/entry_points.txt +2 -0
- csv2sef-1.0.0.dist-info/licenses/LICENSE +21 -0
- csv2sef-1.0.0.dist-info/top_level.txt +1 -0
csv2sef/__init__.py
ADDED
csv2sef/checks.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Range checks (in SI units, by region) and cross-variable checks."""
|
|
3
|
+
import datetime as dt
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
|
|
6
|
+
from . import tables
|
|
7
|
+
from .convert import Context
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def range_check(records, region, log):
|
|
11
|
+
limits = tables.ranges(region)
|
|
12
|
+
for r in records:
|
|
13
|
+
q = r["quantity"]
|
|
14
|
+
x = r["iso_value"]
|
|
15
|
+
if not q or q not in limits or not isinstance(x, (int, float)):
|
|
16
|
+
continue
|
|
17
|
+
lim = limits[q]
|
|
18
|
+
flag = None
|
|
19
|
+
if (lim["out_min"] is not None and x < lim["out_min"]) or \
|
|
20
|
+
(lim["out_max"] is not None and x > lim["out_max"]):
|
|
21
|
+
flag = "out of range"
|
|
22
|
+
elif lim["low"] is not None and x < lim["low"]:
|
|
23
|
+
flag = "low value"
|
|
24
|
+
elif lim["high"] is not None and x > lim["high"]:
|
|
25
|
+
flag = "high value"
|
|
26
|
+
if flag:
|
|
27
|
+
r["flag"] = "; ".join(filter(None, [r["flag"], flag]))
|
|
28
|
+
log.write(Context(r["row"], r["column"], r["date"], r["var"]), "RANGE",
|
|
29
|
+
f"{r['orig_value']} -> {x} {r['iso_unit']}: {flag} ({region} limits)")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def cross_checks(records, log, tn_night_after=False):
|
|
33
|
+
by_key = defaultdict(dict)
|
|
34
|
+
for r in records:
|
|
35
|
+
if isinstance(r["iso_value"], (int, float)):
|
|
36
|
+
by_key[(r["date"], r["hour"], r["minute"])][r["var"]] = r
|
|
37
|
+
|
|
38
|
+
def note(rec, msg, flag=None):
|
|
39
|
+
log.write(Context(rec["row"], rec["column"], rec["date"], rec["var"]), "CROSS", msg)
|
|
40
|
+
if flag:
|
|
41
|
+
rec["flag"] = "; ".join(filter(None, [rec["flag"], flag]))
|
|
42
|
+
|
|
43
|
+
for (date, h, mi), v in by_key.items():
|
|
44
|
+
if h is None:
|
|
45
|
+
continue
|
|
46
|
+
if "tb" in v and "tdb" in v and v["tb"]["iso_value"] > v["tdb"]["iso_value"] + 0.3:
|
|
47
|
+
note(v["tb"], f"Wet bulb {v['tb']['orig_value']} > dry bulb {v['tdb']['orig_value']}",
|
|
48
|
+
"wet bulb > dry bulb")
|
|
49
|
+
if "ta" in v and "tdb" in v and abs(v["ta"]["iso_value"] - v["tdb"]["iso_value"]) > 3:
|
|
50
|
+
note(v["ta"], f"Air temperature {v['ta']['orig_value']} differs from dry bulb "
|
|
51
|
+
f"{v['tdb']['orig_value']}")
|
|
52
|
+
|
|
53
|
+
day_ta = defaultdict(list)
|
|
54
|
+
for (date, h, mi), v in by_key.items():
|
|
55
|
+
if h is not None and "ta" in v:
|
|
56
|
+
day_ta[date].append(v["ta"])
|
|
57
|
+
for (date, h, mi), v in by_key.items():
|
|
58
|
+
if h is not None:
|
|
59
|
+
continue
|
|
60
|
+
for mn, mx in (("Tn", "Tx"),):
|
|
61
|
+
if mn in v and mx in v and v[mn]["iso_value"] > v[mx]["iso_value"]:
|
|
62
|
+
note(v[mn], f"Minimum {v[mn]['orig_value']} > maximum {v[mx]['orig_value']}")
|
|
63
|
+
if "Tn" in v and day_ta.get(date):
|
|
64
|
+
if tn_night_after:
|
|
65
|
+
nxt = date + dt.timedelta(days=1)
|
|
66
|
+
cands = [r for r in day_ta[date] if r["hour"] >= 12] + \
|
|
67
|
+
[r for r in day_ta.get(nxt, []) if r["hour"] < 12]
|
|
68
|
+
else:
|
|
69
|
+
cands = day_ta[date]
|
|
70
|
+
if cands:
|
|
71
|
+
low = min(cands, key=lambda r: r["iso_value"])
|
|
72
|
+
if v["Tn"]["iso_value"] > low["iso_value"] + 0.5:
|
|
73
|
+
note(v["Tn"], f"Minimum {v['Tn']['orig_value']} > {low['column']} "
|
|
74
|
+
f"{low['orig_value']} on {low['date']}")
|
|
75
|
+
if "Tx" in v and day_ta.get(date):
|
|
76
|
+
high = max(day_ta[date], key=lambda r: r["iso_value"])
|
|
77
|
+
if v["Tx"]["iso_value"] < high["iso_value"] - 0.5:
|
|
78
|
+
note(v["Tx"], f"Maximum {v['Tx']['orig_value']} < {high['column']} {high['orig_value']}")
|
csv2sef/cli.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Command line interface.
|
|
4
|
+
|
|
5
|
+
csv2sef check DATA METADATA dry run: how each column will be read
|
|
6
|
+
csv2sef convert DATA METADATA [-o DIR] write clean/ISO csv, log and SEF files
|
|
7
|
+
csv2sef template [DATA] [-o FILE] write a blank metadata file
|
|
8
|
+
csv2sef validate SEF_FILE ... check SEF files against SEF 1.0.0
|
|
9
|
+
csv2sef variables list known variables and units
|
|
10
|
+
"""
|
|
11
|
+
import argparse
|
|
12
|
+
import importlib.metadata
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
# from . import CITATION
|
|
17
|
+
from . import metadata as mdmod
|
|
18
|
+
from . import pipeline, reader, sef, tables
|
|
19
|
+
from .headers import parse_header
|
|
20
|
+
|
|
21
|
+
# from importlib.metadata
|
|
22
|
+
|
|
23
|
+
CITATION = ("Slonosky, V., Black, R., Podolsky, L., Wang, X. & Cheng, V. (2026). "
|
|
24
|
+
"Transcribing historical Canadian weather data. Scientific Data 13, 678. "
|
|
25
|
+
"https://doi.org/10.1038/s41597-025-06036-y")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def cmd_check(a):
|
|
29
|
+
print(pipeline.run(a.data, a.metadata, check_only=True, local_time=a.local))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def cmd_convert(a):
|
|
33
|
+
print(pipeline.run(a.data, a.metadata, outdir=a.outdir, local_time=a.local))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def cmd_template(a):
|
|
37
|
+
found = []
|
|
38
|
+
if a.data:
|
|
39
|
+
t = reader.read(a.data)
|
|
40
|
+
cols, julian = reader.date_columns(t.headers)
|
|
41
|
+
known = tables.variables()
|
|
42
|
+
seen = set()
|
|
43
|
+
for i, hd in enumerate(t.headers):
|
|
44
|
+
if i in cols.values() or i == julian or not hd:
|
|
45
|
+
continue
|
|
46
|
+
h = parse_header(hd)
|
|
47
|
+
if h.var in seen:
|
|
48
|
+
continue
|
|
49
|
+
seen.add(h.var)
|
|
50
|
+
if h.unit_token:
|
|
51
|
+
continue # unit already in the header
|
|
52
|
+
d = known.get(h.var)
|
|
53
|
+
found.append((h.var, d["unit"] if d else "", d["description"] if d else "UNKNOWN variable"))
|
|
54
|
+
out = a.output or "metadata.csv"
|
|
55
|
+
if os.path.exists(out) and not a.force:
|
|
56
|
+
sys.exit(f"{out} exists; use --force to overwrite")
|
|
57
|
+
mdmod.write_template(out, found)
|
|
58
|
+
print(f"Wrote {out}. Fill in the site details"
|
|
59
|
+
+ (" and check the unit.<variable> lines." if found else "."))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def cmd_validate(a):
|
|
63
|
+
bad = 0
|
|
64
|
+
for p in a.files:
|
|
65
|
+
probs = sef.validate(p)
|
|
66
|
+
print(f"{p}: {'OK' if not probs else 'INVALID'}")
|
|
67
|
+
for x in probs[:10]:
|
|
68
|
+
print(" ", x)
|
|
69
|
+
bad += bool(probs)
|
|
70
|
+
sys.exit(1 if bad else 0)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def cmd_variables(a):
|
|
74
|
+
print(f"{'code':11} {'default unit':13} {'stat':8} description")
|
|
75
|
+
for code, d in tables.variables().items():
|
|
76
|
+
print(f"{code:11} {d['unit']:13} {d['stat']:8} {d['description']}")
|
|
77
|
+
print("\nUnits (write in the header in [ ], e.g. 'ta 07:00 [F]'):")
|
|
78
|
+
for u, desc in tables.unit_descriptions().items():
|
|
79
|
+
print(f" {u:13} {desc}")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def main(argv=None):
|
|
83
|
+
p = argparse.ArgumentParser(
|
|
84
|
+
prog="csv2sef",
|
|
85
|
+
description=f"csv2sef {importlib.metadata.metadata(__package__)['Version']}: historical weather csv/Excel -> SEF 1.0.0. "
|
|
86
|
+
f"Authors: {', '.join(importlib.metadata.metadata(__package__)['Author-email'])}.",
|
|
87
|
+
epilog="Please cite: " + CITATION)
|
|
88
|
+
p.add_argument("--version", action="version", version=f"csv2sef {importlib.metadata.metadata(__package__)['Version']}")
|
|
89
|
+
sub = p.add_subparsers(dest="cmd")
|
|
90
|
+
|
|
91
|
+
for name, fn, hlp in (("check", cmd_check, "dry run: show how each column will be read"),
|
|
92
|
+
("convert", cmd_convert, "convert and write SEF files")):
|
|
93
|
+
s = sub.add_parser(name, help=hlp)
|
|
94
|
+
s.add_argument("data", help="csv, tsv or xlsx file")
|
|
95
|
+
s.add_argument("metadata", help="metadata csv (or json)")
|
|
96
|
+
s.add_argument("--local", action="store_true", help="keep local times (no UTC conversion)")
|
|
97
|
+
if name == "convert":
|
|
98
|
+
s.add_argument("-o", "--outdir", help="output directory")
|
|
99
|
+
s.set_defaults(func=fn)
|
|
100
|
+
|
|
101
|
+
s = sub.add_parser("template", help="write a blank metadata file")
|
|
102
|
+
s.add_argument("data", nargs="?", help="optional data file: adds unit lines for its variables")
|
|
103
|
+
s.add_argument("-o", "--output", help="output file (default metadata.csv)")
|
|
104
|
+
s.add_argument("--force", action="store_true")
|
|
105
|
+
s.set_defaults(func=cmd_template)
|
|
106
|
+
|
|
107
|
+
s = sub.add_parser("validate", help="validate SEF files")
|
|
108
|
+
s.add_argument("files", nargs="+")
|
|
109
|
+
s.set_defaults(func=cmd_validate)
|
|
110
|
+
|
|
111
|
+
s = sub.add_parser("variables", help="list known variables and units")
|
|
112
|
+
s.set_defaults(func=cmd_variables)
|
|
113
|
+
|
|
114
|
+
argv = sys.argv[1:] if argv is None else argv
|
|
115
|
+
# shorthand: csv2sef DATA METADATA == convert
|
|
116
|
+
if len(argv) >= 2 and argv[0] not in sub.choices and not argv[0].startswith("-"):
|
|
117
|
+
argv = ["convert"] + argv
|
|
118
|
+
a = p.parse_args(argv)
|
|
119
|
+
if not getattr(a, "func", None):
|
|
120
|
+
p.print_help()
|
|
121
|
+
return
|
|
122
|
+
a.func(a)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
if __name__ == "__main__":
|
|
126
|
+
main()
|