csv2sef 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csv2sef/__init__.py +5 -0
- csv2sef/checks.py +78 -0
- csv2sef/cli.py +126 -0
- csv2sef/convert.py +591 -0
- csv2sef/data/ranges.csv +17 -0
- csv2sef/data/units.csv +91 -0
- csv2sef/data/variable_aliases.csv +73 -0
- csv2sef/data/variables.csv +80 -0
- csv2sef/data/weather_terms.csv +71 -0
- csv2sef/data/wind_words.csv +40 -0
- csv2sef/headers.py +89 -0
- csv2sef/metadata.py +245 -0
- csv2sef/pipeline.py +485 -0
- csv2sef/reader.py +135 -0
- csv2sef/sef.py +116 -0
- csv2sef/tables.py +101 -0
- csv2sef-1.0.0.dist-info/METADATA +41 -0
- csv2sef-1.0.0.dist-info/RECORD +22 -0
- csv2sef-1.0.0.dist-info/WHEEL +5 -0
- csv2sef-1.0.0.dist-info/entry_points.txt +2 -0
- csv2sef-1.0.0.dist-info/licenses/LICENSE +21 -0
- csv2sef-1.0.0.dist-info/top_level.txt +1 -0
csv2sef/sef.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Writing and validating Station Exchange Format (SEF) 1.0.0 files.
|
|
4
|
+
|
|
5
|
+
Structure: 12 header lines (SEF, ID, Name, Lat, Lon, Alt, Source, Link, Vbl, Stat,
|
|
6
|
+
Units, Meta), a column line (Year Month Day Hour Minute Period Value Meta) and one
|
|
7
|
+
tab-separated line per observation. UTF-8, .tsv extension.
|
|
8
|
+
"""
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
SEF_VERSION = "1.0.0"
|
|
12
|
+
HEADER_FIELDS = ["SEF", "ID", "Name", "Lat", "Lon", "Alt", "Source", "Link",
|
|
13
|
+
"Vbl", "Stat", "Units", "Meta"]
|
|
14
|
+
COLUMNS = ["Year", "Month", "Day", "Hour", "Minute", "Period", "Value", "Meta"]
|
|
15
|
+
TEXT_UNITS = {"text", "synoptic code", "cloud atlas code"}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def clean_text(s):
|
|
19
|
+
return str(s).replace("\t", " ").replace("|", "/").replace("\r", " ").replace("\n", " ").strip()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _value(v, missing):
|
|
23
|
+
if v is None:
|
|
24
|
+
return missing
|
|
25
|
+
if isinstance(v, float):
|
|
26
|
+
return f"{v:g}" if (abs(v) >= 1e-3 or v == 0) else f"{v:.4f}"
|
|
27
|
+
return clean_text(v)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def write(path, site, vbl, stat, units, header_meta, rows, missing="NA"):
|
|
31
|
+
"""rows: dicts with year, month, day, hour, minute, period, value, meta (list of str)."""
|
|
32
|
+
with open(path, "w", encoding="utf-8", newline="\n") as f:
|
|
33
|
+
head = [SEF_VERSION, site["id"], clean_text(site["name"]), site["lat"], site["lon"],
|
|
34
|
+
site["alt"], clean_text(site.get("source", "")), clean_text(site.get("link", "")),
|
|
35
|
+
vbl, stat, units, header_meta]
|
|
36
|
+
for k, v in zip(HEADER_FIELDS, head):
|
|
37
|
+
f.write(f"{k}\t{v}\n")
|
|
38
|
+
f.write("\t".join(COLUMNS) + "\n")
|
|
39
|
+
for r in rows:
|
|
40
|
+
na = lambda x: "NA" if x is None else str(x)
|
|
41
|
+
f.write("\t".join([str(r["year"]), str(r["month"]), str(r["day"]), na(r["hour"]),
|
|
42
|
+
na(r["minute"]), str(r["period"]), _value(r["value"], missing),
|
|
43
|
+
"|".join(clean_text(m) for m in r["meta"])]) + "\n")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def filename(site, vbl, start, end, suffix=""):
|
|
47
|
+
"""<source>_<id>_<YYYY-MM>_<YYYY-MM>_<vbl>.tsv (as in dataresqc::write_sef)."""
|
|
48
|
+
src = re.sub(r"[^A-Za-z0-9._-]", "", site.get("source", "")) or "NA"
|
|
49
|
+
return f"{src}_{site['id']}_{start:%Y-%m}_{end:%Y-%m}_{vbl}{suffix}.tsv"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def validate(path):
|
|
53
|
+
"""Return a list of problems (empty list = valid SEF 1.0.0)."""
|
|
54
|
+
problems = []
|
|
55
|
+
try:
|
|
56
|
+
with open(path, encoding="utf-8") as f:
|
|
57
|
+
lines = f.read().split("\n")
|
|
58
|
+
except UnicodeDecodeError:
|
|
59
|
+
return ["file is not UTF-8"]
|
|
60
|
+
if lines and lines[-1] == "":
|
|
61
|
+
lines = lines[:-1]
|
|
62
|
+
if len(lines) < 13:
|
|
63
|
+
return [f"only {len(lines)} lines; need 12 header lines and a column line"]
|
|
64
|
+
head = {}
|
|
65
|
+
for i, name in enumerate(HEADER_FIELDS):
|
|
66
|
+
parts = lines[i].split("\t")
|
|
67
|
+
if parts[0] != name or len(parts) != 2:
|
|
68
|
+
problems.append(f"line {i + 1}: expected '{name}<TAB>value', got '{lines[i][:40]}'")
|
|
69
|
+
else:
|
|
70
|
+
head[name] = parts[1]
|
|
71
|
+
if head.get("SEF") != SEF_VERSION:
|
|
72
|
+
problems.append(f"SEF version is '{head.get('SEF')}', expected {SEF_VERSION}")
|
|
73
|
+
if "ID" in head and not re.fullmatch(r"[A-Za-z0-9._-]+", head["ID"]):
|
|
74
|
+
problems.append("ID may only contain letters, digits, '-', '_' and '.'")
|
|
75
|
+
for k, lo, hi in (("Lat", -90, 90), ("Lon", -180, 360)):
|
|
76
|
+
try:
|
|
77
|
+
x = float(head.get(k, "x"))
|
|
78
|
+
if not lo <= x <= hi:
|
|
79
|
+
problems.append(f"{k} {x} out of range")
|
|
80
|
+
except ValueError:
|
|
81
|
+
problems.append(f"{k} '{head.get(k)}' is not a number")
|
|
82
|
+
if head.get("Alt", "NA") not in ("NA", ""):
|
|
83
|
+
try:
|
|
84
|
+
float(head["Alt"])
|
|
85
|
+
except ValueError:
|
|
86
|
+
problems.append(f"Alt '{head['Alt']}' is not a number")
|
|
87
|
+
if lines[12].split("\t") != COLUMNS:
|
|
88
|
+
problems.append("line 13 must be: " + " ".join(COLUMNS))
|
|
89
|
+
textual = head.get("Units", "") in TEXT_UNITS
|
|
90
|
+
for n, line in enumerate(lines[13:], start=14):
|
|
91
|
+
c = line.split("\t")
|
|
92
|
+
if len(c) != 8:
|
|
93
|
+
problems.append(f"line {n}: {len(c)} columns instead of 8")
|
|
94
|
+
continue
|
|
95
|
+
try:
|
|
96
|
+
y, mo = int(c[0]), c[1]
|
|
97
|
+
if mo not in ("", "NA") and not 1 <= int(mo) <= 12:
|
|
98
|
+
raise ValueError
|
|
99
|
+
if c[2] not in ("", "NA") and not 1 <= int(c[2]) <= 31:
|
|
100
|
+
raise ValueError
|
|
101
|
+
if c[3] not in ("", "NA") and not 0 <= int(c[3]) <= 24:
|
|
102
|
+
raise ValueError
|
|
103
|
+
if c[4] not in ("", "NA") and not 0 <= int(c[4]) <= 59:
|
|
104
|
+
raise ValueError
|
|
105
|
+
except ValueError:
|
|
106
|
+
problems.append(f"line {n}: invalid date/time {c[:5]}")
|
|
107
|
+
if not textual and c[6] not in ("", "NA"):
|
|
108
|
+
try:
|
|
109
|
+
float(c[6])
|
|
110
|
+
except ValueError:
|
|
111
|
+
if c[6] not in ("Calm", "Variable"):
|
|
112
|
+
problems.append(f"line {n}: value '{c[6]}' is not numeric")
|
|
113
|
+
if len(problems) > 50:
|
|
114
|
+
problems.append("... (stopped after 50 problems)")
|
|
115
|
+
break
|
|
116
|
+
return problems
|
csv2sef/tables.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Lookup tables shipped in csv2sef/data/ (editable csv files), plus user tables."""
|
|
3
|
+
import csv
|
|
4
|
+
import os
|
|
5
|
+
from functools import lru_cache
|
|
6
|
+
|
|
7
|
+
DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def read_table(path):
|
|
11
|
+
"""Read a csv lookup table; lines starting with '#' are comments."""
|
|
12
|
+
with open(path, encoding="utf-8-sig", newline="") as f:
|
|
13
|
+
lines = [ln for ln in f if ln.strip() and not ln.lstrip().startswith("#")]
|
|
14
|
+
return [{k.strip(): (v or "").strip() for k, v in row.items() if k}
|
|
15
|
+
for row in csv.DictReader(lines)]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _data(name):
|
|
19
|
+
return read_table(os.path.join(DATA_DIR, name))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@lru_cache(maxsize=None)
|
|
23
|
+
def variables():
|
|
24
|
+
"""code -> dict(description, unit, stat, period, quantity, sef_vbl)"""
|
|
25
|
+
return {r["code"]: r for r in _data("variables.csv")}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@lru_cache(maxsize=None)
|
|
29
|
+
def unit_aliases():
|
|
30
|
+
"""lower-case alias -> canonical unit code"""
|
|
31
|
+
out = {}
|
|
32
|
+
for r in _data("units.csv"):
|
|
33
|
+
out[r["alias"].lower()] = r["unit"]
|
|
34
|
+
out.setdefault(r["unit"].lower(), r["unit"])
|
|
35
|
+
return out
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@lru_cache(maxsize=None)
|
|
39
|
+
def unit_descriptions():
|
|
40
|
+
out = {}
|
|
41
|
+
for r in _data("units.csv"):
|
|
42
|
+
out.setdefault(r["unit"], r["description"])
|
|
43
|
+
return out
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@lru_cache(maxsize=None)
|
|
47
|
+
def variable_aliases():
|
|
48
|
+
"""normalised alias -> code (longest first)"""
|
|
49
|
+
out = {norm_name(r["alias"]): r["code"] for r in _data("variable_aliases.csv")}
|
|
50
|
+
return dict(sorted(out.items(), key=lambda kv: -len(kv[0])))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def norm_name(s):
|
|
54
|
+
import re
|
|
55
|
+
s = s.replace("_", " ").replace("'", "").replace('"', "").lower()
|
|
56
|
+
return re.sub(r"\s+", " ", s).strip()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def canonical_unit(token):
|
|
60
|
+
"""'°F' -> 'F', 'in Hg' -> 'inHg'; None if unknown."""
|
|
61
|
+
if token is None:
|
|
62
|
+
return None
|
|
63
|
+
t = " ".join(token.replace("\xa0", " ").split()).lower()
|
|
64
|
+
if t in ("", "na", "n/a", "none", "-"):
|
|
65
|
+
return None
|
|
66
|
+
return unit_aliases().get(t)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def wind_words(extra_files=()):
|
|
70
|
+
"""term (lower case) -> Beaufort number or 'unknown'. User files take precedence."""
|
|
71
|
+
out = {}
|
|
72
|
+
rows = []
|
|
73
|
+
for path in extra_files:
|
|
74
|
+
rows += read_table(path)
|
|
75
|
+
rows += _data("wind_words.csv")
|
|
76
|
+
for r in rows:
|
|
77
|
+
term = r["term"].lower()
|
|
78
|
+
if term and term not in out:
|
|
79
|
+
out[term] = r["beaufort"]
|
|
80
|
+
return out
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def weather_terms(extra_files=()):
|
|
84
|
+
"""Ordered list of (term, code, match). User rows come first."""
|
|
85
|
+
rows = []
|
|
86
|
+
for path in extra_files:
|
|
87
|
+
rows += read_table(path)
|
|
88
|
+
rows += _data("weather_terms.csv")
|
|
89
|
+
return [(r["term"].lower(), r["code"], (r.get("match") or "contains").lower())
|
|
90
|
+
for r in rows if r.get("term")]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def ranges(region="global"):
|
|
94
|
+
"""quantity -> dict(out_min, low, high, out_max) as floats/None."""
|
|
95
|
+
def f(x):
|
|
96
|
+
return float(x) if x not in ("", None) else None
|
|
97
|
+
rows = [r for r in _data("ranges.csv") if r["region"].lower() == region.lower()]
|
|
98
|
+
if not rows:
|
|
99
|
+
raise ValueError(f"Unknown range region '{region}' (see data/ranges.csv)")
|
|
100
|
+
return {r["quantity"]: {k: f(r[k]) for k in ("out_min", "low", "high", "out_max")}
|
|
101
|
+
for r in rows}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: csv2sef
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: convert transcribed historical weather observations (csv / Excel) into the Station Exchange Format (SEF 1.0.0).
|
|
5
|
+
Author-email: Vicky Slonosky <VickyS08@users.noreply.github.com>, Hervé Hacot <hhacot@users.noreply.github.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/open-data-rescue/csv2sef
|
|
8
|
+
Project-URL: Issues, https://github.com/open-data-rescue/csv2sef/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.14
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: cdrutilities>=1.0.1
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# csv2sef
|
|
18
|
+
csv2sef - convert transcribed historical weather observations (csv / Excel)
|
|
19
|
+
into the Station Exchange Format (SEF 1.0.0).
|
|
20
|
+
|
|
21
|
+
Authors: Hervé Hacot, Victoria Slonosky (Open Data Rescue)
|
|
22
|
+
|
|
23
|
+
Please cite:
|
|
24
|
+
Slonosky, V., Black, R., Podolsky, L., Wang, X. & Cheng, V. (2026).
|
|
25
|
+
Transcribing historical Canadian weather data. Scientific Data 13, 678.
|
|
26
|
+
https://doi.org/10.1038/s41597-025-06036-y
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
|
|
30
|
+
If using this repository, ensure that you have uv installed
|
|
31
|
+
|
|
32
|
+
1. clone the repo
|
|
33
|
+
2. run `uv sync`
|
|
34
|
+
3. build using `uv build`
|
|
35
|
+
4. run csv2sef using
|
|
36
|
+
```
|
|
37
|
+
uv run csv2sef
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Development
|
|
41
|
+
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
csv2sef/__init__.py,sha256=EJ98lpcoRamkDRkRHfp-MU7e4c-8yv8GrLuKTwwoEYU,48
|
|
2
|
+
csv2sef/checks.py,sha256=vkD44dhRDRWanbg6dCtpkC7XY3kdwbIRUfzZkgGYiSU,3487
|
|
3
|
+
csv2sef/cli.py,sha256=APtvKGTWlux1bJ0ulgDX73og_34l31jUBWt7neGvzX4,4803
|
|
4
|
+
csv2sef/convert.py,sha256=rxHLRQBjwxyOSZpX5zFebCmik_ZjQYVZa4mfufhQ0rE,22750
|
|
5
|
+
csv2sef/headers.py,sha256=onkG6-n18yc8UWYl97AYlrcVf95q3y2puard0WkORfk,3121
|
|
6
|
+
csv2sef/metadata.py,sha256=ferpG7ZK_rnBTwn41gJY9lJG5ZBpZcjcJ6ekEj70pio,10481
|
|
7
|
+
csv2sef/pipeline.py,sha256=fZQ9QTHRPMsXWt_hJRBtuLTpObbFnQNWfe1qlTU4gi8,23404
|
|
8
|
+
csv2sef/reader.py,sha256=u_h6ZS3WwaKRA_KoeloRej-Nj5GQ3bHNFYDRp_v7xYo,4826
|
|
9
|
+
csv2sef/sef.py,sha256=DlNMzx6ezB-kDw3uh4b6dFXKgXuozbQ3pxms477W3Hs,4958
|
|
10
|
+
csv2sef/tables.py,sha256=dlFLG4lyYGu1atcoHkIVM0T2kQN-JjP-SR4XlS2j8bk,3162
|
|
11
|
+
csv2sef/data/ranges.csv,sha256=E0zDE1K_9SAiJIS-xtkN2LAVHquBIPCyWBV5i6ZilBg,767
|
|
12
|
+
csv2sef/data/units.csv,sha256=thKjmiaqX4Ccfum59hcph4XiKGNtSOarQ0ophXhPhOU,3679
|
|
13
|
+
csv2sef/data/variable_aliases.csv,sha256=QfL7rzucA8xsBEIknd6-vGb_fgXOKkDnKYoI_EkbDdw,1531
|
|
14
|
+
csv2sef/data/variables.csv,sha256=pZQySUA-ny6T15pcgjMwuNb8zr3wGJAVoFhhCxDKMjI,3947
|
|
15
|
+
csv2sef/data/weather_terms.csv,sha256=p1PIjRCEkRdPvFkqPTVg1bTDTOR4g5xoVJfyMb-HIlE,1947
|
|
16
|
+
csv2sef/data/wind_words.csv,sha256=BN1nOgcF8SQy0GiMRtAFLg5TN2FLlH0BOiPHO9CMmYc,803
|
|
17
|
+
csv2sef-1.0.0.dist-info/licenses/LICENSE,sha256=Mt84dPHtoOMyVhl5oJiOvohG9rDiZ_7X-112mI-fK3A,1073
|
|
18
|
+
csv2sef-1.0.0.dist-info/METADATA,sha256=IdCClE6k-OJT5Cl9M8k2h5saEMiJkuPPTba2HmjG6qo,1290
|
|
19
|
+
csv2sef-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
20
|
+
csv2sef-1.0.0.dist-info/entry_points.txt,sha256=CLmUf9PYkxl-S-8N0DUXXhJuaWdlNGaUtdrauP1ATUE,41
|
|
21
|
+
csv2sef-1.0.0.dist-info/top_level.txt,sha256=q1K9t3YVyR67gXAjjjGG_torjyG7D8YmRTBbmH03jEc,8
|
|
22
|
+
csv2sef-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Open Data Rescue
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
csv2sef
|