csv2sef 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
csv2sef/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ from .cli import main
2
+
3
+ __all__ = [
4
+ "main"
5
+ ]
csv2sef/checks.py ADDED
@@ -0,0 +1,78 @@
1
+ # -*- coding: utf-8 -*-
2
+ """Range checks (in SI units, by region) and cross-variable checks."""
3
+ import datetime as dt
4
+ from collections import defaultdict
5
+
6
+ from . import tables
7
+ from .convert import Context
8
+
9
+
10
+ def range_check(records, region, log):
11
+ limits = tables.ranges(region)
12
+ for r in records:
13
+ q = r["quantity"]
14
+ x = r["iso_value"]
15
+ if not q or q not in limits or not isinstance(x, (int, float)):
16
+ continue
17
+ lim = limits[q]
18
+ flag = None
19
+ if (lim["out_min"] is not None and x < lim["out_min"]) or \
20
+ (lim["out_max"] is not None and x > lim["out_max"]):
21
+ flag = "out of range"
22
+ elif lim["low"] is not None and x < lim["low"]:
23
+ flag = "low value"
24
+ elif lim["high"] is not None and x > lim["high"]:
25
+ flag = "high value"
26
+ if flag:
27
+ r["flag"] = "; ".join(filter(None, [r["flag"], flag]))
28
+ log.write(Context(r["row"], r["column"], r["date"], r["var"]), "RANGE",
29
+ f"{r['orig_value']} -> {x} {r['iso_unit']}: {flag} ({region} limits)")
30
+
31
+
32
+ def cross_checks(records, log, tn_night_after=False):
33
+ by_key = defaultdict(dict)
34
+ for r in records:
35
+ if isinstance(r["iso_value"], (int, float)):
36
+ by_key[(r["date"], r["hour"], r["minute"])][r["var"]] = r
37
+
38
+ def note(rec, msg, flag=None):
39
+ log.write(Context(rec["row"], rec["column"], rec["date"], rec["var"]), "CROSS", msg)
40
+ if flag:
41
+ rec["flag"] = "; ".join(filter(None, [rec["flag"], flag]))
42
+
43
+ for (date, h, mi), v in by_key.items():
44
+ if h is None:
45
+ continue
46
+ if "tb" in v and "tdb" in v and v["tb"]["iso_value"] > v["tdb"]["iso_value"] + 0.3:
47
+ note(v["tb"], f"Wet bulb {v['tb']['orig_value']} > dry bulb {v['tdb']['orig_value']}",
48
+ "wet bulb > dry bulb")
49
+ if "ta" in v and "tdb" in v and abs(v["ta"]["iso_value"] - v["tdb"]["iso_value"]) > 3:
50
+ note(v["ta"], f"Air temperature {v['ta']['orig_value']} differs from dry bulb "
51
+ f"{v['tdb']['orig_value']}")
52
+
53
+ day_ta = defaultdict(list)
54
+ for (date, h, mi), v in by_key.items():
55
+ if h is not None and "ta" in v:
56
+ day_ta[date].append(v["ta"])
57
+ for (date, h, mi), v in by_key.items():
58
+ if h is not None:
59
+ continue
60
+ for mn, mx in (("Tn", "Tx"),):
61
+ if mn in v and mx in v and v[mn]["iso_value"] > v[mx]["iso_value"]:
62
+ note(v[mn], f"Minimum {v[mn]['orig_value']} > maximum {v[mx]['orig_value']}")
63
+ if "Tn" in v and day_ta.get(date):
64
+ if tn_night_after:
65
+ nxt = date + dt.timedelta(days=1)
66
+ cands = [r for r in day_ta[date] if r["hour"] >= 12] + \
67
+ [r for r in day_ta.get(nxt, []) if r["hour"] < 12]
68
+ else:
69
+ cands = day_ta[date]
70
+ if cands:
71
+ low = min(cands, key=lambda r: r["iso_value"])
72
+ if v["Tn"]["iso_value"] > low["iso_value"] + 0.5:
73
+ note(v["Tn"], f"Minimum {v['Tn']['orig_value']} > {low['column']} "
74
+ f"{low['orig_value']} on {low['date']}")
75
+ if "Tx" in v and day_ta.get(date):
76
+ high = max(day_ta[date], key=lambda r: r["iso_value"])
77
+ if v["Tx"]["iso_value"] < high["iso_value"] - 0.5:
78
+ note(v["Tx"], f"Maximum {v['Tx']['orig_value']} < {high['column']} {high['orig_value']}")
csv2sef/cli.py ADDED
@@ -0,0 +1,126 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Command line interface.
4
+
5
+ csv2sef check DATA METADATA dry run: how each column will be read
6
+ csv2sef convert DATA METADATA [-o DIR] write clean/ISO csv, log and SEF files
7
+ csv2sef template [DATA] [-o FILE] write a blank metadata file
8
+ csv2sef validate SEF_FILE ... check SEF files against SEF 1.0.0
9
+ csv2sef variables list known variables and units
10
+ """
11
+ import argparse
12
+ import importlib.metadata
13
+ import os
14
+ import sys
15
+
16
+ # from . import CITATION
17
+ from . import metadata as mdmod
18
+ from . import pipeline, reader, sef, tables
19
+ from .headers import parse_header
20
+
21
+ # from importlib.metadata
22
+
23
+ CITATION = ("Slonosky, V., Black, R., Podolsky, L., Wang, X. & Cheng, V. (2026). "
24
+ "Transcribing historical Canadian weather data. Scientific Data 13, 678. "
25
+ "https://doi.org/10.1038/s41597-025-06036-y")
26
+
27
+
28
+ def cmd_check(a):
29
+ print(pipeline.run(a.data, a.metadata, check_only=True, local_time=a.local))
30
+
31
+
32
+ def cmd_convert(a):
33
+ print(pipeline.run(a.data, a.metadata, outdir=a.outdir, local_time=a.local))
34
+
35
+
36
+ def cmd_template(a):
37
+ found = []
38
+ if a.data:
39
+ t = reader.read(a.data)
40
+ cols, julian = reader.date_columns(t.headers)
41
+ known = tables.variables()
42
+ seen = set()
43
+ for i, hd in enumerate(t.headers):
44
+ if i in cols.values() or i == julian or not hd:
45
+ continue
46
+ h = parse_header(hd)
47
+ if h.var in seen:
48
+ continue
49
+ seen.add(h.var)
50
+ if h.unit_token:
51
+ continue # unit already in the header
52
+ d = known.get(h.var)
53
+ found.append((h.var, d["unit"] if d else "", d["description"] if d else "UNKNOWN variable"))
54
+ out = a.output or "metadata.csv"
55
+ if os.path.exists(out) and not a.force:
56
+ sys.exit(f"{out} exists; use --force to overwrite")
57
+ mdmod.write_template(out, found)
58
+ print(f"Wrote {out}. Fill in the site details"
59
+ + (" and check the unit.<variable> lines." if found else "."))
60
+
61
+
62
+ def cmd_validate(a):
63
+ bad = 0
64
+ for p in a.files:
65
+ probs = sef.validate(p)
66
+ print(f"{p}: {'OK' if not probs else 'INVALID'}")
67
+ for x in probs[:10]:
68
+ print(" ", x)
69
+ bad += bool(probs)
70
+ sys.exit(1 if bad else 0)
71
+
72
+
73
+ def cmd_variables(a):
74
+ print(f"{'code':11} {'default unit':13} {'stat':8} description")
75
+ for code, d in tables.variables().items():
76
+ print(f"{code:11} {d['unit']:13} {d['stat']:8} {d['description']}")
77
+ print("\nUnits (write in the header in [ ], e.g. 'ta 07:00 [F]'):")
78
+ for u, desc in tables.unit_descriptions().items():
79
+ print(f" {u:13} {desc}")
80
+
81
+
82
+ def main(argv=None):
83
+ p = argparse.ArgumentParser(
84
+ prog="csv2sef",
85
+ description=f"csv2sef {importlib.metadata.metadata(__package__)['Version']}: historical weather csv/Excel -> SEF 1.0.0. "
86
+ f"Authors: {', '.join(importlib.metadata.metadata(__package__)['Author-email'])}.",
87
+ epilog="Please cite: " + CITATION)
88
+ p.add_argument("--version", action="version", version=f"csv2sef {importlib.metadata.metadata(__package__)['Version']}")
89
+ sub = p.add_subparsers(dest="cmd")
90
+
91
+ for name, fn, hlp in (("check", cmd_check, "dry run: show how each column will be read"),
92
+ ("convert", cmd_convert, "convert and write SEF files")):
93
+ s = sub.add_parser(name, help=hlp)
94
+ s.add_argument("data", help="csv, tsv or xlsx file")
95
+ s.add_argument("metadata", help="metadata csv (or json)")
96
+ s.add_argument("--local", action="store_true", help="keep local times (no UTC conversion)")
97
+ if name == "convert":
98
+ s.add_argument("-o", "--outdir", help="output directory")
99
+ s.set_defaults(func=fn)
100
+
101
+ s = sub.add_parser("template", help="write a blank metadata file")
102
+ s.add_argument("data", nargs="?", help="optional data file: adds unit lines for its variables")
103
+ s.add_argument("-o", "--output", help="output file (default metadata.csv)")
104
+ s.add_argument("--force", action="store_true")
105
+ s.set_defaults(func=cmd_template)
106
+
107
+ s = sub.add_parser("validate", help="validate SEF files")
108
+ s.add_argument("files", nargs="+")
109
+ s.set_defaults(func=cmd_validate)
110
+
111
+ s = sub.add_parser("variables", help="list known variables and units")
112
+ s.set_defaults(func=cmd_variables)
113
+
114
+ argv = sys.argv[1:] if argv is None else argv
115
+ # shorthand: csv2sef DATA METADATA == convert
116
+ if len(argv) >= 2 and argv[0] not in sub.choices and not argv[0].startswith("-"):
117
+ argv = ["convert"] + argv
118
+ a = p.parse_args(argv)
119
+ if not getattr(a, "func", None):
120
+ p.print_help()
121
+ return
122
+ a.func(a)
123
+
124
+
125
+ if __name__ == "__main__":
126
+ main()