csv2sef 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csv2sef/__init__.py +5 -0
- csv2sef/checks.py +78 -0
- csv2sef/cli.py +126 -0
- csv2sef/convert.py +591 -0
- csv2sef/data/ranges.csv +17 -0
- csv2sef/data/units.csv +91 -0
- csv2sef/data/variable_aliases.csv +73 -0
- csv2sef/data/variables.csv +80 -0
- csv2sef/data/weather_terms.csv +71 -0
- csv2sef/data/wind_words.csv +40 -0
- csv2sef/headers.py +89 -0
- csv2sef/metadata.py +245 -0
- csv2sef/pipeline.py +485 -0
- csv2sef/reader.py +135 -0
- csv2sef/sef.py +116 -0
- csv2sef/tables.py +101 -0
- csv2sef-1.0.0.dist-info/METADATA +41 -0
- csv2sef-1.0.0.dist-info/RECORD +22 -0
- csv2sef-1.0.0.dist-info/WHEEL +5 -0
- csv2sef-1.0.0.dist-info/entry_points.txt +2 -0
- csv2sef-1.0.0.dist-info/licenses/LICENSE +21 -0
- csv2sef-1.0.0.dist-info/top_level.txt +1 -0
csv2sef/convert.py
ADDED
|
@@ -0,0 +1,591 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Screening, clean-up and conversion of single transcribed values to SI units.
|
|
4
|
+
|
|
5
|
+
Ported from transcription_data_processing.py (Open Data Rescue) and Tables 3-9 of
|
|
6
|
+
Slonosky et al. (2026). Unit conversions use lmrlib.py (NOAA ICOADS).
|
|
7
|
+
|
|
8
|
+
process_value(raw, var, unit, ctx, log, opts) -> (flag, clean_value, iso_value, iso_unit)
|
|
9
|
+
"""
|
|
10
|
+
import re
|
|
11
|
+
from collections import Counter
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from fractions import Fraction
|
|
14
|
+
from math import sqrt
|
|
15
|
+
|
|
16
|
+
from CDRUtilities import iso_mapping as im
|
|
17
|
+
from CDRUtilities import lmrlib as lmr
|
|
18
|
+
|
|
19
|
+
CM_VARS = {"sd", "swe", "fs"} # snow depth, water equivalent, fresh snow: written in cm
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# --------------------------------------------------------------------------
|
|
23
|
+
@dataclass
|
|
24
|
+
class Context:
|
|
25
|
+
row: object # line number in the input file
|
|
26
|
+
column: str
|
|
27
|
+
date: object
|
|
28
|
+
var: str
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Logger:
|
|
32
|
+
"""Tab-separated validation log."""
|
|
33
|
+
|
|
34
|
+
def __init__(self, path=None):
|
|
35
|
+
self.f = open(path, "w", encoding="utf-8") if path else None
|
|
36
|
+
if self.f:
|
|
37
|
+
self.f.write("date\trow\tcolumn\tcategory\tmessage\n")
|
|
38
|
+
self.counts = Counter()
|
|
39
|
+
|
|
40
|
+
def write(self, ctx, category, msg):
|
|
41
|
+
self.counts[category] += 1
|
|
42
|
+
if self.f:
|
|
43
|
+
d = ctx.date.isoformat() if ctx and ctx.date else ""
|
|
44
|
+
self.f.write(f"{d}\t{ctx.row if ctx else ''}\t{ctx.column if ctx else ''}\t{category}\t{msg}\n")
|
|
45
|
+
|
|
46
|
+
def info(self, msg):
|
|
47
|
+
if self.f:
|
|
48
|
+
self.f.write(f"\t\t\tINFO\t{msg}\n")
|
|
49
|
+
|
|
50
|
+
def close(self):
|
|
51
|
+
if self.f:
|
|
52
|
+
self.f.close()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# --------------------------------------------------------------------------
|
|
56
|
+
# Missing / invalid values (Table 7)
|
|
57
|
+
# --------------------------------------------------------------------------
|
|
58
|
+
MISSING_EXACT = {"", "-", "--", "m", "na", "n/a", "nan", "missing", "none"}
|
|
59
|
+
MISSING_TERMS = ["not.taken", "not taken", "unknown symbol", "retracted", "-999",
|
|
60
|
+
"no grass", "no place", "suspended", "abt on duty"]
|
|
61
|
+
INSTRUMENT_ERROR_TERMS = ["out of order", "out.of.order", "broken", "unserviceable", "-888",
|
|
62
|
+
"incorrect", "not reliable", "covered in snow", "covered with snow",
|
|
63
|
+
"no error", "frozen in position", "instrument error"]
|
|
64
|
+
BELOW_SCALE_TERMS = ["mercury in bulb", "mercury in the bulb", "mercury inside bulb",
|
|
65
|
+
"in the ball", "dans la boule", "frozen mercury", "mercury frozen"]
|
|
66
|
+
DITTO_MARKS = {'"', "'", "''", '""', "~", "”", "〃", "do", "do."}
|
|
67
|
+
TEXT_UNITS = {"mno", "text", "lct", "uct", "dir", "words", "oz"}
|
|
68
|
+
# explicit "empty"/"dot" typed by a transcriber = nothing to observe (legacy behaviour)
|
|
69
|
+
EMPTY_TOKEN_VALUE = {"Sm": "0", "Bf": "0", "mph": "0", "kt": "0", "m/s": "0", "km/h": "0",
|
|
70
|
+
"lbsft": "0", "words": "calm", "cloudvel": "0", "tenths": "0", "okta": "0",
|
|
71
|
+
"in": "0", "mm": "0", "cm": "0", "dir": "calm", "lct": "0", "uct": "0"}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def pre_screen(value, var, unit, ctx, log):
|
|
75
|
+
v = (value or "").replace("\xa0", " ").strip()
|
|
76
|
+
vl = v.lower()
|
|
77
|
+
if vl in ("empty", "dot"):
|
|
78
|
+
if unit in EMPTY_TOKEN_VALUE and var != "eee":
|
|
79
|
+
return "ok", EMPTY_TOKEN_VALUE[unit], "no entry"
|
|
80
|
+
return "stop", None, "missing"
|
|
81
|
+
if vl in MISSING_EXACT:
|
|
82
|
+
return "stop", None, "missing"
|
|
83
|
+
if unit == "text":
|
|
84
|
+
return "ok", v, None # free text is kept verbatim
|
|
85
|
+
if vl == "*":
|
|
86
|
+
log.write(ctx, "INSTRUMENT", "'*' - value not able to be recorded")
|
|
87
|
+
return "stop", None, "instrument error"
|
|
88
|
+
if vl in DITTO_MARKS:
|
|
89
|
+
log.write(ctx, "DITTO", f"Ditto mark '{v}' - value not reconstructed")
|
|
90
|
+
return "stop", None, "ditto"
|
|
91
|
+
if all(ch in "?. " for ch in vl):
|
|
92
|
+
log.write(ctx, "ILLEGIBLE", f"'{v}'")
|
|
93
|
+
return "stop", None, "illegible"
|
|
94
|
+
if unit != "mno" and any(x in vl for x in BELOW_SCALE_TERMS):
|
|
95
|
+
log.write(ctx, "SCALE", f"'{v}': below the thermometer scale")
|
|
96
|
+
return "stop", None, "below scale" # value set in process_value if below_scale_c
|
|
97
|
+
if unit != "mno" and any(x in vl for x in INSTRUMENT_ERROR_TERMS):
|
|
98
|
+
log.write(ctx, "INSTRUMENT", f"Instrument error: '{v}'")
|
|
99
|
+
return "stop", None, "instrument error"
|
|
100
|
+
if "illegible" in vl and "cirrus" not in vl:
|
|
101
|
+
log.write(ctx, "ILLEGIBLE", f"'{v}'")
|
|
102
|
+
return "stop", None, "illegible"
|
|
103
|
+
if unit not in TEXT_UNITS and any(x in vl for x in MISSING_TERMS):
|
|
104
|
+
return "stop", None, "missing"
|
|
105
|
+
if unit not in TEXT_UNITS and "?" in v: # e.g. "37.?" -> 37, flagged uncertain
|
|
106
|
+
stripped = v.replace("?", "").strip().rstrip(".")
|
|
107
|
+
if re.search(r"\d", stripped):
|
|
108
|
+
log.write(ctx, "UNCERTAIN", f"'{v}' read as '{stripped}'")
|
|
109
|
+
return "ok", stripped, "uncertain"
|
|
110
|
+
log.write(ctx, "ILLEGIBLE", f"'{v}'")
|
|
111
|
+
return "stop", None, "illegible"
|
|
112
|
+
return "ok", v, None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# --------------------------------------------------------------------------
|
|
116
|
+
# Numeric clean-up (Table 8)
|
|
117
|
+
# --------------------------------------------------------------------------
|
|
118
|
+
def clean_number(value, ctx, log):
|
|
119
|
+
v = value.strip()
|
|
120
|
+
orig = v
|
|
121
|
+
v = v.replace("[", "").replace("]", "")
|
|
122
|
+
if v.startswith("--"):
|
|
123
|
+
v = "-" + v.lstrip("-")
|
|
124
|
+
v = v.replace("- ", "-").replace("+ ", "").lstrip("+")
|
|
125
|
+
if "/" in v: # fractions: "37 1/2", "37.1/2"
|
|
126
|
+
v = v.replace(".", " ").replace(",", " ")
|
|
127
|
+
neg = v.startswith("-")
|
|
128
|
+
res = float(sum(Fraction(s) for s in v.lstrip("-").split()))
|
|
129
|
+
res = -res if neg else res
|
|
130
|
+
else:
|
|
131
|
+
v = v.replace(",", ".").replace(" ", ".")
|
|
132
|
+
while ".." in v:
|
|
133
|
+
v = v.replace("..", ".")
|
|
134
|
+
v = v.rstrip(".")
|
|
135
|
+
res = float(v)
|
|
136
|
+
if v != orig:
|
|
137
|
+
log.write(ctx, "SYNTAX", f"'{orig}' cleaned to {res:g}")
|
|
138
|
+
return res
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _r(x, nd=2):
|
|
142
|
+
return None if x is None else round(float(x), nd)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# --------------------------------------------------------------------------
|
|
146
|
+
# Converters: (v, var, ctx, log, opts) -> (flag, clean_value, iso_value)
|
|
147
|
+
# --------------------------------------------------------------------------
|
|
148
|
+
# size of one degree in degrees Celsius, and conversion to Celsius, per temperature unit
|
|
149
|
+
DEGREE_C = {"F": 5.0 / 9.0, "C": 1.0, "R": 1.25, "De": 2.0 / 3.0, "De_frz": 2.0 / 3.0}
|
|
150
|
+
TO_C = {"F": lambda x: lmr.temp_f2c(x), "C": lambda x: x, "R": lambda x: lmr.temp_r2c(x),
|
|
151
|
+
"De": lambda x: 100.0 - x * 2.0 / 3.0, # Delisle: 0 = boiling, 150 = freezing
|
|
152
|
+
"De_frz": lambda x: x * 2.0 / 3.0} # Delisle degrees counted from freezing
|
|
153
|
+
SUFFIX_UNIT = {"r": "R", "f": "F", "c": "C", "de": "De"}
|
|
154
|
+
NUM = r"(\d+(?:[.,]\d+)?)"
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _num(x):
|
|
158
|
+
return float(x.replace(",", "."))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def temperature(v, unit, ctx, log, opts):
|
|
162
|
+
"""Temperature with historical notation -> (flag, clean value, degrees C).
|
|
163
|
+
|
|
164
|
+
'16R' / '58F' value in the unit given by the letter (overrides the column unit)
|
|
165
|
+
'10+co', '2-c' 10 degrees above / 2 below the Caves de l'Observatoire (Paris),
|
|
166
|
+
taken as caves_temperature_c (default 12.8 C), whatever the scale of the reading; 'co', '=c' = caves
|
|
167
|
+
'<=-34' at or below -34 (in C if below_scale_c is set, else in the column unit)
|
|
168
|
+
"""
|
|
169
|
+
s = v.strip().lower().replace("°", "").replace("º", "").replace(" ", "")
|
|
170
|
+
s = s.replace("(", "").replace(")", "").strip("'\"") # '+5 (R)', "-28 R'"
|
|
171
|
+
if s.startswith("+"):
|
|
172
|
+
s = s[1:]
|
|
173
|
+
caves = float(opts.get("caves_temperature_c", 12.8))
|
|
174
|
+
s = s.replace("c.o.", "co").replace("c.o", "co")
|
|
175
|
+
CAVES = r"(?:co?|o|0)" # c, co, c.o, o or 0 after a sign = the caves
|
|
176
|
+
m = re.fullmatch(NUM + r"([+-])(" + CAVES + r")?(r|f|de)?", s)
|
|
177
|
+
if m and (m.group(3) or not m.group(4)):
|
|
178
|
+
x = _num(m.group(1))
|
|
179
|
+
u = SUFFIX_UNIT.get(m.group(4) or "", unit)
|
|
180
|
+
c = caves + (x if m.group(2) == "+" else -x) * DEGREE_C[u]
|
|
181
|
+
flag = "caves notation" if m.group(3) else "caves notation (assumed)"
|
|
182
|
+
log.write(ctx, "NOTATION", f"'{v}' = {m.group(2)}{x:g} {u} from the caves ({caves:g} C) -> {c:.2f} C")
|
|
183
|
+
return flag, v, _r(c)
|
|
184
|
+
if re.fullmatch(r"(?:0?=)?" + CAVES + r"(?:r|f)?", s) and s not in ("0",):
|
|
185
|
+
log.write(ctx, "NOTATION", f"'{v}' = caves temperature {caves:g} C")
|
|
186
|
+
return "caves notation", v, _r(caves)
|
|
187
|
+
m = re.fullmatch(NUM + r"=" + CAVES + r"(r|f|de)?", s)
|
|
188
|
+
if m:
|
|
189
|
+
x = _num(m.group(1))
|
|
190
|
+
if x == 0:
|
|
191
|
+
return "caves notation", v, _r(caves)
|
|
192
|
+
log.write(ctx, "NOTATION", f"'{v}': {x:g} degrees from the caves, sign unclear - not converted")
|
|
193
|
+
return "caves notation, sign unclear", v, None
|
|
194
|
+
m = re.fullmatch(r"(<=?|>=?|≤|≥)(-?\d+(?:[.,]\d+)?)([a-z]*)", s)
|
|
195
|
+
if m:
|
|
196
|
+
x = _num(m.group(2))
|
|
197
|
+
flag = "at or below" if m.group(1) in ("<", "<=", "≤") else "at or above"
|
|
198
|
+
u = SUFFIX_UNIT.get(m.group(3))
|
|
199
|
+
if u is None:
|
|
200
|
+
u = "C" if opts.get("below_scale_c") not in (None, "") else unit
|
|
201
|
+
log.write(ctx, "LIMIT", f"'{v}': {flag} {x:g} {u}")
|
|
202
|
+
return flag, x, _r(TO_C[u](x))
|
|
203
|
+
m = re.fullmatch(r"(-?\d+(?:[.,]\d+)?)(r|f|c|de)", s)
|
|
204
|
+
if m:
|
|
205
|
+
u = SUFFIX_UNIT[m.group(2)]
|
|
206
|
+
x = _num(m.group(1))
|
|
207
|
+
if u != unit:
|
|
208
|
+
log.write(ctx, "UNIT", f"'{v}' read in {u} (column unit {unit})")
|
|
209
|
+
return (f"unit {u}" if u != unit else None), x, _r(TO_C[u](x))
|
|
210
|
+
x = clean_number(v, ctx, log)
|
|
211
|
+
return None, x, _r(TO_C[unit](x))
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def c_F(v, var, ctx, log, opts):
|
|
215
|
+
return temperature(v, "F", ctx, log, opts)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def c_C(v, var, ctx, log, opts):
|
|
219
|
+
return temperature(v, "C", ctx, log, opts)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def c_R(v, var, ctx, log, opts):
|
|
223
|
+
return temperature(v, "R", ctx, log, opts)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def c_De(v, var, ctx, log, opts):
|
|
227
|
+
return temperature(v, "De", ctx, log, opts)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def c_De_frz(v, var, ctx, log, opts):
|
|
231
|
+
return temperature(v, "De_frz", ctx, log, opts)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def c_Fdiff(v, var, ctx, log, opts):
|
|
235
|
+
t = clean_number(v, ctx, log)
|
|
236
|
+
return None, t, _r(t * 5.0 / 9.0)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def c_Cdiff(v, var, ctx, log, opts):
|
|
240
|
+
t = clean_number(v, ctx, log)
|
|
241
|
+
return None, t, _r(t)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _vapour_decimal(v, p, var, ctx, log):
|
|
245
|
+
if var.startswith("e") and var != "eee" and "." not in v and p > 1:
|
|
246
|
+
log.write(ctx, "SYNTAX", f"Vapour pressure '{v}' read as {p / 1000:g}")
|
|
247
|
+
return p / 1000.0, "missing decimal"
|
|
248
|
+
return p, None
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def c_inHg(v, var, ctx, log, opts):
|
|
252
|
+
p = clean_number(v, ctx, log)
|
|
253
|
+
p, flag = _vapour_decimal(v, p, var, ctx, log)
|
|
254
|
+
return flag, p, _r(lmr.baro_Eng_in2mb(p))
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def c_mmHg(v, var, ctx, log, opts):
|
|
258
|
+
p = clean_number(v, ctx, log)
|
|
259
|
+
p, flag = _vapour_decimal(v, p, var, ctx, log)
|
|
260
|
+
return flag, p, _r(lmr.baro_mm2mb(p))
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def c_hPa(v, var, ctx, log, opts):
|
|
264
|
+
p = clean_number(v, ctx, log)
|
|
265
|
+
return None, p, _r(p)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def c_ParisIn(v, var, ctx, log, opts):
|
|
269
|
+
"""French inches, optionally with lines: '27 11.5' or '27p 11.5l' = 27 in 11.5 lines."""
|
|
270
|
+
nums = re.findall(r"\d+(?:[.,]\d+)?", v)
|
|
271
|
+
if not nums:
|
|
272
|
+
raise ValueError
|
|
273
|
+
inches = float(nums[0].replace(",", "."))
|
|
274
|
+
if len(nums) > 1:
|
|
275
|
+
inches += float(nums[1].replace(",", ".")) / 12.0
|
|
276
|
+
return None, round(inches, 4), _r(lmr.baro_Fr_in2mb(inches))
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def c_ParisLine(v, var, ctx, log, opts):
|
|
280
|
+
lines = clean_number(v, ctx, log)
|
|
281
|
+
return None, lines, _r(lmr.baro_Fr_in2mb(lines / 12.0))
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
TRACE_TERMS = ["trace", "slight", "meas", "inapp", "in."]
|
|
285
|
+
NIL_TERMS = {"nil", "no rain", "no snow"}
|
|
286
|
+
LITTLE_TERMS = {"a little", "little", "some", "a few", "un peu", "peu", "quelque"}
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _precip(v, var, ctx, log, opts, to_mm):
|
|
290
|
+
"""Precipitation / snow; returns mm (later scaled to cm for snow)."""
|
|
291
|
+
vl = v.lower().strip()
|
|
292
|
+
if vl in NIL_TERMS:
|
|
293
|
+
return "no entry", 0.0, 0.0
|
|
294
|
+
if any(x in vl for x in TRACE_TERMS) or vl in ("in", "t", "tr"):
|
|
295
|
+
mm = float(opts.get("trace_mm", 0.2))
|
|
296
|
+
return "trace", round(mm / to_mm, 4), _r(mm)
|
|
297
|
+
if vl in LITTLE_TERMS:
|
|
298
|
+
mm = float(opts.get("trace_mm", 0.2))
|
|
299
|
+
log.write(ctx, "QUALITATIVE", f"'{v}': treated as trace")
|
|
300
|
+
return "trace; qualitative", v, _r(mm)
|
|
301
|
+
if not re.search(r"\d", vl):
|
|
302
|
+
# words only ("Rain", "snow"): occurrence noted, amount unknown
|
|
303
|
+
log.write(ctx, "QUALITATIVE", f"'{v}': amount not measured")
|
|
304
|
+
return "qualitative", v, None
|
|
305
|
+
s = re.sub(r"^[RrSs]\s+", "", v) # leading R (rain) / S (snow) markers
|
|
306
|
+
flag = None
|
|
307
|
+
if "|" in s or " / " in s or "+" in s:
|
|
308
|
+
parts = [p for p in re.split(r"\||\s/\s|\+", s) if p.strip()]
|
|
309
|
+
res = sum(clean_number(p, ctx, log) for p in parts)
|
|
310
|
+
flag = "two values added"
|
|
311
|
+
log.write(ctx, "SUM", f"'{v}' -> {res:g}")
|
|
312
|
+
else:
|
|
313
|
+
res = clean_number(s, ctx, log)
|
|
314
|
+
return flag, res, _r(res * to_mm)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def c_in(v, var, ctx, log, opts):
|
|
318
|
+
return _precip(v, var, ctx, log, opts, 25.4)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def c_mm(v, var, ctx, log, opts):
|
|
322
|
+
return _precip(v, var, ctx, log, opts, 1.0)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def c_cm(v, var, ctx, log, opts):
|
|
326
|
+
return _precip(v, var, ctx, log, opts, 10.0)
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def c_mi(v, var, ctx, log, opts):
|
|
330
|
+
d = clean_number(v, ctx, log)
|
|
331
|
+
return None, d, _r(d * 1609.344, 0)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def c_percent(v, var, ctx, log, opts):
|
|
335
|
+
r = clean_number(v, ctx, log)
|
|
336
|
+
if 0 < r <= 1:
|
|
337
|
+
return "multiply by 100", r, _r(r * 100)
|
|
338
|
+
return None, r, _r(r)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _scale(v, ctx, log, top, name):
|
|
342
|
+
from fractions import Fraction as Fr
|
|
343
|
+
s = int(float(sum(Fr(x) for x in v.replace(",", " ").split())))
|
|
344
|
+
if s < 0 or s > top:
|
|
345
|
+
log.write(ctx, "RANGE", f"{name} {s}: out of range")
|
|
346
|
+
return s, False
|
|
347
|
+
return s, True
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def c_Sm(v, var, ctx, log, opts):
|
|
351
|
+
if not re.search(r"\d", v):
|
|
352
|
+
return c_words(v, var, ctx, log, opts) # descriptive term in a numeric column
|
|
353
|
+
s, ok = _scale(v, ctx, log, 10, "Smithsonian force")
|
|
354
|
+
if not ok:
|
|
355
|
+
return "out of range", s, None
|
|
356
|
+
return None, s, _r(im.convertSmithsonian(s), 1) # Table 4: nominal mph -> m/s
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def c_Bf(v, var, ctx, log, opts):
|
|
360
|
+
if not re.search(r"\d", v):
|
|
361
|
+
return c_words(v, var, ctx, log, opts)
|
|
362
|
+
b, ok = _scale(v, ctx, log, 12, "Beaufort force")
|
|
363
|
+
if not ok:
|
|
364
|
+
return "out of range", b, None
|
|
365
|
+
return None, b, _r(im.convertBeaufort(b), 1)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def c_words(v, var, ctx, log, opts):
|
|
369
|
+
"""Descriptive wind force, looked up in wind_words tables."""
|
|
370
|
+
words = opts["wind_words"]
|
|
371
|
+
key = " ".join(v.lower().split())
|
|
372
|
+
if key in ("calm", "0", "c"):
|
|
373
|
+
return None, "calm", 0.0
|
|
374
|
+
bf = words.get(key)
|
|
375
|
+
if bf is None: # longest table term inside the text
|
|
376
|
+
for term in sorted(words, key=len, reverse=True):
|
|
377
|
+
if re.search(r"(?<!\w)" + re.escape(term) + r"(?!\w)", key):
|
|
378
|
+
bf = words[term]
|
|
379
|
+
log.write(ctx, "KEYWORD", f"Wind force '{v}' matched '{term}'")
|
|
380
|
+
break
|
|
381
|
+
if bf is None or str(bf).strip() == "":
|
|
382
|
+
log.write(ctx, "UNMAPPED", f"Wind force term '{v}' not in wind_words tables")
|
|
383
|
+
return "unmapped", v, None
|
|
384
|
+
if str(bf).lower() == "qualitative":
|
|
385
|
+
log.write(ctx, "QUALITATIVE", f"Wind force term '{v}' kept as qualitative (no scale equivalent)")
|
|
386
|
+
return "qualitative", v, None
|
|
387
|
+
if str(bf).lower() == "unknown":
|
|
388
|
+
log.write(ctx, "UNKNOWN", f"Wind force term '{v}' coded as unknown")
|
|
389
|
+
return "unknown", v, None
|
|
390
|
+
return None, v, _r(im.convertBeaufort(int(float(bf))), 1)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def c_mph(v, var, ctx, log, opts):
|
|
394
|
+
s = clean_number(v, ctx, log)
|
|
395
|
+
return None, s, _r(s * 0.44704)
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def c_kt(v, var, ctx, log, opts):
|
|
399
|
+
s = clean_number(v, ctx, log)
|
|
400
|
+
return None, s, _r(lmr.wind_kts2mps(s))
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def c_ms(v, var, ctx, log, opts):
|
|
404
|
+
s = clean_number(v, ctx, log)
|
|
405
|
+
return None, s, _r(s)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def c_kmh(v, var, ctx, log, opts):
|
|
409
|
+
s = clean_number(v, ctx, log)
|
|
410
|
+
return None, s, _r(s / 3.6)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def c_lbsft(v, var, ctx, log, opts):
|
|
414
|
+
"""Wind pressure lbs/ft2 ('3 15' = 3 lb 15 oz) -> m/s (Equation 1)."""
|
|
415
|
+
parts = v.split()
|
|
416
|
+
w = float(parts[0]) + float(parts[1]) / 16.0 if len(parts) == 2 else clean_number(v, ctx, log)
|
|
417
|
+
if w < 0:
|
|
418
|
+
return "out of range", w, None
|
|
419
|
+
return None, w, _r(sqrt(w / 0.00256) * 0.44704)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
OBSCURED = ["fog", "smoke", "haze", "scud", "hidden", "mist"]
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def c_tenths(v, var, ctx, log, opts):
|
|
426
|
+
vl = v.lower()
|
|
427
|
+
if any(x in vl for x in OBSCURED):
|
|
428
|
+
return None, v, 9
|
|
429
|
+
if "clear" in vl or vl == "zero":
|
|
430
|
+
return None, 0, 0
|
|
431
|
+
t = clean_number(v, ctx, log)
|
|
432
|
+
if t < 0 or t > 10:
|
|
433
|
+
log.write(ctx, "RANGE", f"Cloud amount {t:g} tenths: out of range")
|
|
434
|
+
return "out of range", t, None
|
|
435
|
+
return None, t, _r(t * 0.8)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def c_okta(v, var, ctx, log, opts):
|
|
439
|
+
vl = v.lower()
|
|
440
|
+
if any(x in vl for x in OBSCURED):
|
|
441
|
+
return None, v, 9
|
|
442
|
+
t = clean_number(v, ctx, log)
|
|
443
|
+
return None, t, _r(t)
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def c_tenths_clear(v, var, ctx, log, opts):
|
|
447
|
+
"""'Clearness of sky' in tenths of CLEAR sky -> oktas of covered sky."""
|
|
448
|
+
vl = v.lower()
|
|
449
|
+
if any(x in vl for x in OBSCURED):
|
|
450
|
+
return None, v, 9
|
|
451
|
+
if "clear" in vl:
|
|
452
|
+
return None, 10, 0
|
|
453
|
+
t = clean_number(v, ctx, log)
|
|
454
|
+
if t < 0 or t > 10:
|
|
455
|
+
log.write(ctx, "RANGE", f"Clearness {t:g}: out of range")
|
|
456
|
+
return "out of range", t, None
|
|
457
|
+
return None, t, lmr.cloud_tenthsclear2oktas(int(round(t)))
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def c_cloudvel(v, var, ctx, log, opts):
|
|
461
|
+
s = v.split()[-1]
|
|
462
|
+
if s.lower() in ("p", "perceptible"):
|
|
463
|
+
return None, v, 1
|
|
464
|
+
c = clean_number(s, ctx, log)
|
|
465
|
+
if c < 0 or c > 10:
|
|
466
|
+
log.write(ctx, "RANGE", f"Cloud velocity {c:g}: out of range")
|
|
467
|
+
return "out of range", c, None
|
|
468
|
+
return None, c, _r(c, 0)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
ABBR_TO_DEG = {abbr.lower(): im.convert_directions[full]
|
|
472
|
+
for full, abbr in im.abbr_directions.items()
|
|
473
|
+
if full in im.convert_directions and im.convert_directions[full] != -999}
|
|
474
|
+
ABBR_TO_DEG.update({k.lower(): val for k, val in im.convert_directions.items() if val != -999})
|
|
475
|
+
ABBR_TO_DEG.update({"northeast": 45, "northwest": 315, "southeast": 135, "southwest": 225,
|
|
476
|
+
"nxe": 11.25, "exn": 78.75, "nwbyn": 326.25,
|
|
477
|
+
# French / German compass letters
|
|
478
|
+
"o": 270, "no": 315, "so": 225, "ono": 292.5, "oso": 247.5,
|
|
479
|
+
"nno": 337.5, "sso": 202.5})
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def c_dir(v, var, ctx, log, opts):
|
|
483
|
+
vl = v.strip().lower().replace(".", "").replace(" by ", "by")
|
|
484
|
+
if vl in ("calm", "c", "0", "none", "not perceptible", "imp", "calme"):
|
|
485
|
+
return None, v, "Calm"
|
|
486
|
+
if vl.startswith("var"):
|
|
487
|
+
return None, v, "Variable"
|
|
488
|
+
try:
|
|
489
|
+
d = float(vl)
|
|
490
|
+
if 0 <= d <= 360:
|
|
491
|
+
return None, v, _r(d % 360, 1)
|
|
492
|
+
except ValueError:
|
|
493
|
+
pass
|
|
494
|
+
first = vl.split()[0] if vl.split() else vl
|
|
495
|
+
for key in (vl, vl.replace(" ", "-"), first):
|
|
496
|
+
if key in ABBR_TO_DEG:
|
|
497
|
+
return None, v, ABBR_TO_DEG[key]
|
|
498
|
+
log.write(ctx, "UNMAPPED", f"Direction '{v}' not recognised")
|
|
499
|
+
return "unmapped", v, None
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def _cloudtype(v, var, ctx, log, opts, upper):
|
|
503
|
+
if opts.get("keep_wind_cloud_type"):
|
|
504
|
+
code = im.abbr_cloud.get(v.lower(), "-999")
|
|
505
|
+
else:
|
|
506
|
+
code = im.convertUpperCloud(v) if upper else im.convertLowerCloud(v)
|
|
507
|
+
if code == "-999":
|
|
508
|
+
log.write(ctx, "UNMAPPED", f"Cloud type '{v}' not recognised")
|
|
509
|
+
return "unmapped", v, None
|
|
510
|
+
return None, v, code
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def c_lct(v, var, ctx, log, opts):
|
|
514
|
+
return _cloudtype(v, var, ctx, log, opts, False)
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def c_uct(v, var, ctx, log, opts):
|
|
518
|
+
return _cloudtype(v, var, ctx, log, opts, True)
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def c_mno(v, var, ctx, log, opts):
|
|
522
|
+
"""Weather description -> synoptic codes, using the weather_terms tables in order."""
|
|
523
|
+
text = " ".join(v.lower().split())
|
|
524
|
+
codes = []
|
|
525
|
+
for term, code, match in opts["weather_terms"]:
|
|
526
|
+
if match == "exact":
|
|
527
|
+
if text == term:
|
|
528
|
+
codes = [code]
|
|
529
|
+
break
|
|
530
|
+
continue
|
|
531
|
+
pat = (r"(?<!\w)" + re.escape(term) + r"(?!\w)") if match == "word" \
|
|
532
|
+
else (r"(?<!\w)" + re.escape(term) + r"\w*")
|
|
533
|
+
if re.search(pat, text):
|
|
534
|
+
text = re.sub(pat, " ", text)
|
|
535
|
+
for c in code.split():
|
|
536
|
+
if c not in codes:
|
|
537
|
+
codes.append(c)
|
|
538
|
+
if codes == ["unknown"]:
|
|
539
|
+
log.write(ctx, "UNKNOWN", f"Weather term '{v}' coded as unknown")
|
|
540
|
+
return "unknown", v, None
|
|
541
|
+
codes = [c for c in codes if c.lower() != "unknown"]
|
|
542
|
+
if not codes:
|
|
543
|
+
log.write(ctx, "UNMAPPED", f"Weather '{v}' has no synoptic equivalent")
|
|
544
|
+
return "unmapped", v, None
|
|
545
|
+
return None, v, " ".join(codes)
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def c_text(v, var, ctx, log, opts):
|
|
549
|
+
return None, v, v
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
CONVERTERS = {
|
|
553
|
+
"F": (c_F, "C"), "C": (c_C, "C"), "R": (c_R, "C"), "De": (c_De, "C"), "De_frz": (c_De_frz, "C"), "Fdiff": (c_Fdiff, "C"), "Cdiff": (c_Cdiff, "C"),
|
|
554
|
+
"inHg": (c_inHg, "hPa"), "mmHg": (c_mmHg, "hPa"), "hPa": (c_hPa, "hPa"),
|
|
555
|
+
"ParisIn": (c_ParisIn, "hPa"), "ParisLine": (c_ParisLine, "hPa"),
|
|
556
|
+
"in": (c_in, "mm"), "mm": (c_mm, "mm"), "cm": (c_cm, "mm"), "mi": (c_mi, "m"),
|
|
557
|
+
"%": (c_percent, "%"),
|
|
558
|
+
"Sm": (c_Sm, "m/s"), "Bf": (c_Bf, "m/s"), "words": (c_words, "m/s"), "mph": (c_mph, "m/s"),
|
|
559
|
+
"kt": (c_kt, "m/s"), "m/s": (c_ms, "m/s"), "km/h": (c_kmh, "m/s"), "lbsft": (c_lbsft, "m/s"),
|
|
560
|
+
"tenths": (c_tenths, "okta"), "okta": (c_okta, "okta"), "tenths_clear": (c_tenths_clear, "okta"),
|
|
561
|
+
"cloudvel": (c_cloudvel, "0-10"), "dir": (c_dir, "deg"),
|
|
562
|
+
"lct": (c_lct, "cloud atlas code"), "uct": (c_uct, "cloud atlas code"),
|
|
563
|
+
"mno": (c_mno, "synoptic code"), "text": (c_text, "text"), "oz": (c_text, "scale"),
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def iso_unit(unit, var):
|
|
568
|
+
u = CONVERTERS[unit][1] if unit in CONVERTERS else unit
|
|
569
|
+
return "cm" if (u == "mm" and var in CM_VARS) else u
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def process_value(raw, var, unit, ctx, log, opts):
|
|
573
|
+
"""Screen, clean and convert one value -> (flag, clean_value, iso_value, iso_unit)."""
|
|
574
|
+
iu = iso_unit(unit, var)
|
|
575
|
+
status, v, flag = pre_screen(raw, var, unit, ctx, log)
|
|
576
|
+
if status == "stop":
|
|
577
|
+
if flag == "below scale" and iu == "C" and opts.get("below_scale_c") not in (None, ""):
|
|
578
|
+
return "at or below; below scale", None, _r(float(opts["below_scale_c"])), iu
|
|
579
|
+
return flag, None, None, iu
|
|
580
|
+
if unit not in CONVERTERS:
|
|
581
|
+
log.write(ctx, "UNIT", f"No converter for unit '{unit}'")
|
|
582
|
+
return "unknown unit", v, None, iu
|
|
583
|
+
try:
|
|
584
|
+
cflag, clean, iso = CONVERTERS[unit][0](v, var, ctx, log, opts)
|
|
585
|
+
except (ValueError, ZeroDivisionError, IndexError):
|
|
586
|
+
log.write(ctx, "INVALID", f"Invalid data '{raw}'")
|
|
587
|
+
return "invalid", None, None, iu
|
|
588
|
+
if iu == "cm" and isinstance(iso, (int, float)):
|
|
589
|
+
iso = _r(iso / 10.0)
|
|
590
|
+
flags = [f for f in (flag, cflag) if f]
|
|
591
|
+
return ("; ".join(flags) if flags else None), clean, iso, iu
|
csv2sef/data/ranges.csv
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
region,quantity,iso_unit,out_min,low,high,out_max,note
|
|
2
|
+
global,temperature,C,-90,-60,50,60,
|
|
3
|
+
global,pressure,hPa,500,870,1070,1100,station pressure; use canada/regional limits for sea-level stations
|
|
4
|
+
global,vapour,hPa,0,,,80,
|
|
5
|
+
global,humidity,%,0,,,100,
|
|
6
|
+
global,precipitation,mm,0,,300,1000,daily
|
|
7
|
+
global,snow,cm,0,,,1000,
|
|
8
|
+
global,wind,m/s,0,,40,80,
|
|
9
|
+
global,cloud,okta,0,,,9,9 = sky obscured
|
|
10
|
+
canada,temperature,C,-45.5,-42.7,37.8,48.9,Slonosky et al. 2026 Table 9
|
|
11
|
+
canada,pressure,hPa,914.2,965.0,1036.1,1083.5,Slonosky et al. 2026 Table 9
|
|
12
|
+
canada,vapour,hPa,0,,,67.7,Slonosky et al. 2026 Table 9 (2 inHg)
|
|
13
|
+
canada,humidity,%,0,,,100,
|
|
14
|
+
canada,precipitation,mm,0,,,762,Slonosky et al. 2026 Table 9 (30 in)
|
|
15
|
+
canada,snow,cm,0,,,500,
|
|
16
|
+
canada,wind,m/s,0,,40,80,
|
|
17
|
+
canada,cloud,okta,0,,,9,
|