csv2sef 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
csv2sef/sef.py ADDED
@@ -0,0 +1,116 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Writing and validating Station Exchange Format (SEF) 1.0.0 files.
4
+
5
+ Structure: 12 header lines (SEF, ID, Name, Lat, Lon, Alt, Source, Link, Vbl, Stat,
6
+ Units, Meta), a column line (Year Month Day Hour Minute Period Value Meta) and one
7
+ tab-separated line per observation. UTF-8, .tsv extension.
8
+ """
9
+ import re
10
+
11
+ SEF_VERSION = "1.0.0"
12
+ HEADER_FIELDS = ["SEF", "ID", "Name", "Lat", "Lon", "Alt", "Source", "Link",
13
+ "Vbl", "Stat", "Units", "Meta"]
14
+ COLUMNS = ["Year", "Month", "Day", "Hour", "Minute", "Period", "Value", "Meta"]
15
+ TEXT_UNITS = {"text", "synoptic code", "cloud atlas code"}
16
+
17
+
18
+ def clean_text(s):
19
+ return str(s).replace("\t", " ").replace("|", "/").replace("\r", " ").replace("\n", " ").strip()
20
+
21
+
22
+ def _value(v, missing):
23
+ if v is None:
24
+ return missing
25
+ if isinstance(v, float):
26
+ return f"{v:g}" if (abs(v) >= 1e-3 or v == 0) else f"{v:.4f}"
27
+ return clean_text(v)
28
+
29
+
30
+ def write(path, site, vbl, stat, units, header_meta, rows, missing="NA"):
31
+ """rows: dicts with year, month, day, hour, minute, period, value, meta (list of str)."""
32
+ with open(path, "w", encoding="utf-8", newline="\n") as f:
33
+ head = [SEF_VERSION, site["id"], clean_text(site["name"]), site["lat"], site["lon"],
34
+ site["alt"], clean_text(site.get("source", "")), clean_text(site.get("link", "")),
35
+ vbl, stat, units, header_meta]
36
+ for k, v in zip(HEADER_FIELDS, head):
37
+ f.write(f"{k}\t{v}\n")
38
+ f.write("\t".join(COLUMNS) + "\n")
39
+ for r in rows:
40
+ na = lambda x: "NA" if x is None else str(x)
41
+ f.write("\t".join([str(r["year"]), str(r["month"]), str(r["day"]), na(r["hour"]),
42
+ na(r["minute"]), str(r["period"]), _value(r["value"], missing),
43
+ "|".join(clean_text(m) for m in r["meta"])]) + "\n")
44
+
45
+
46
+ def filename(site, vbl, start, end, suffix=""):
47
+ """<source>_<id>_<YYYY-MM>_<YYYY-MM>_<vbl>.tsv (as in dataresqc::write_sef)."""
48
+ src = re.sub(r"[^A-Za-z0-9._-]", "", site.get("source", "")) or "NA"
49
+ return f"{src}_{site['id']}_{start:%Y-%m}_{end:%Y-%m}_{vbl}{suffix}.tsv"
50
+
51
+
52
+ def validate(path):
53
+ """Return a list of problems (empty list = valid SEF 1.0.0)."""
54
+ problems = []
55
+ try:
56
+ with open(path, encoding="utf-8") as f:
57
+ lines = f.read().split("\n")
58
+ except UnicodeDecodeError:
59
+ return ["file is not UTF-8"]
60
+ if lines and lines[-1] == "":
61
+ lines = lines[:-1]
62
+ if len(lines) < 13:
63
+ return [f"only {len(lines)} lines; need 12 header lines and a column line"]
64
+ head = {}
65
+ for i, name in enumerate(HEADER_FIELDS):
66
+ parts = lines[i].split("\t")
67
+ if parts[0] != name or len(parts) != 2:
68
+ problems.append(f"line {i + 1}: expected '{name}<TAB>value', got '{lines[i][:40]}'")
69
+ else:
70
+ head[name] = parts[1]
71
+ if head.get("SEF") != SEF_VERSION:
72
+ problems.append(f"SEF version is '{head.get('SEF')}', expected {SEF_VERSION}")
73
+ if "ID" in head and not re.fullmatch(r"[A-Za-z0-9._-]+", head["ID"]):
74
+ problems.append("ID may only contain letters, digits, '-', '_' and '.'")
75
+ for k, lo, hi in (("Lat", -90, 90), ("Lon", -180, 360)):
76
+ try:
77
+ x = float(head.get(k, "x"))
78
+ if not lo <= x <= hi:
79
+ problems.append(f"{k} {x} out of range")
80
+ except ValueError:
81
+ problems.append(f"{k} '{head.get(k)}' is not a number")
82
+ if head.get("Alt", "NA") not in ("NA", ""):
83
+ try:
84
+ float(head["Alt"])
85
+ except ValueError:
86
+ problems.append(f"Alt '{head['Alt']}' is not a number")
87
+ if lines[12].split("\t") != COLUMNS:
88
+ problems.append("line 13 must be: " + " ".join(COLUMNS))
89
+ textual = head.get("Units", "") in TEXT_UNITS
90
+ for n, line in enumerate(lines[13:], start=14):
91
+ c = line.split("\t")
92
+ if len(c) != 8:
93
+ problems.append(f"line {n}: {len(c)} columns instead of 8")
94
+ continue
95
+ try:
96
+ y, mo = int(c[0]), c[1]
97
+ if mo not in ("", "NA") and not 1 <= int(mo) <= 12:
98
+ raise ValueError
99
+ if c[2] not in ("", "NA") and not 1 <= int(c[2]) <= 31:
100
+ raise ValueError
101
+ if c[3] not in ("", "NA") and not 0 <= int(c[3]) <= 24:
102
+ raise ValueError
103
+ if c[4] not in ("", "NA") and not 0 <= int(c[4]) <= 59:
104
+ raise ValueError
105
+ except ValueError:
106
+ problems.append(f"line {n}: invalid date/time {c[:5]}")
107
+ if not textual and c[6] not in ("", "NA"):
108
+ try:
109
+ float(c[6])
110
+ except ValueError:
111
+ if c[6] not in ("Calm", "Variable"):
112
+ problems.append(f"line {n}: value '{c[6]}' is not numeric")
113
+ if len(problems) > 50:
114
+ problems.append("... (stopped after 50 problems)")
115
+ break
116
+ return problems
csv2sef/tables.py ADDED
@@ -0,0 +1,101 @@
1
+ # -*- coding: utf-8 -*-
2
+ """Lookup tables shipped in csv2sef/data/ (editable csv files), plus user tables."""
3
+ import csv
4
+ import os
5
+ from functools import lru_cache
6
+
7
+ DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
8
+
9
+
10
+ def read_table(path):
11
+ """Read a csv lookup table; lines starting with '#' are comments."""
12
+ with open(path, encoding="utf-8-sig", newline="") as f:
13
+ lines = [ln for ln in f if ln.strip() and not ln.lstrip().startswith("#")]
14
+ return [{k.strip(): (v or "").strip() for k, v in row.items() if k}
15
+ for row in csv.DictReader(lines)]
16
+
17
+
18
+ def _data(name):
19
+ return read_table(os.path.join(DATA_DIR, name))
20
+
21
+
22
+ @lru_cache(maxsize=None)
23
+ def variables():
24
+ """code -> dict(description, unit, stat, period, quantity, sef_vbl)"""
25
+ return {r["code"]: r for r in _data("variables.csv")}
26
+
27
+
28
+ @lru_cache(maxsize=None)
29
+ def unit_aliases():
30
+ """lower-case alias -> canonical unit code"""
31
+ out = {}
32
+ for r in _data("units.csv"):
33
+ out[r["alias"].lower()] = r["unit"]
34
+ out.setdefault(r["unit"].lower(), r["unit"])
35
+ return out
36
+
37
+
38
+ @lru_cache(maxsize=None)
39
+ def unit_descriptions():
40
+ out = {}
41
+ for r in _data("units.csv"):
42
+ out.setdefault(r["unit"], r["description"])
43
+ return out
44
+
45
+
46
+ @lru_cache(maxsize=None)
47
+ def variable_aliases():
48
+ """normalised alias -> code (longest first)"""
49
+ out = {norm_name(r["alias"]): r["code"] for r in _data("variable_aliases.csv")}
50
+ return dict(sorted(out.items(), key=lambda kv: -len(kv[0])))
51
+
52
+
53
+ def norm_name(s):
54
+ import re
55
+ s = s.replace("_", " ").replace("'", "").replace('"', "").lower()
56
+ return re.sub(r"\s+", " ", s).strip()
57
+
58
+
59
+ def canonical_unit(token):
60
+ """'°F' -> 'F', 'in Hg' -> 'inHg'; None if unknown."""
61
+ if token is None:
62
+ return None
63
+ t = " ".join(token.replace("\xa0", " ").split()).lower()
64
+ if t in ("", "na", "n/a", "none", "-"):
65
+ return None
66
+ return unit_aliases().get(t)
67
+
68
+
69
+ def wind_words(extra_files=()):
70
+ """term (lower case) -> Beaufort number or 'unknown'. User files take precedence."""
71
+ out = {}
72
+ rows = []
73
+ for path in extra_files:
74
+ rows += read_table(path)
75
+ rows += _data("wind_words.csv")
76
+ for r in rows:
77
+ term = r["term"].lower()
78
+ if term and term not in out:
79
+ out[term] = r["beaufort"]
80
+ return out
81
+
82
+
83
+ def weather_terms(extra_files=()):
84
+ """Ordered list of (term, code, match). User rows come first."""
85
+ rows = []
86
+ for path in extra_files:
87
+ rows += read_table(path)
88
+ rows += _data("weather_terms.csv")
89
+ return [(r["term"].lower(), r["code"], (r.get("match") or "contains").lower())
90
+ for r in rows if r.get("term")]
91
+
92
+
93
+ def ranges(region="global"):
94
+ """quantity -> dict(out_min, low, high, out_max) as floats/None."""
95
+ def f(x):
96
+ return float(x) if x not in ("", None) else None
97
+ rows = [r for r in _data("ranges.csv") if r["region"].lower() == region.lower()]
98
+ if not rows:
99
+ raise ValueError(f"Unknown range region '{region}' (see data/ranges.csv)")
100
+ return {r["quantity"]: {k: f(r[k]) for k in ("out_min", "low", "high", "out_max")}
101
+ for r in rows}
@@ -0,0 +1,41 @@
1
+ Metadata-Version: 2.4
2
+ Name: csv2sef
3
+ Version: 1.0.0
4
+ Summary: convert transcribed historical weather observations (csv / Excel) into the Station Exchange Format (SEF 1.0.0).
5
+ Author-email: Vicky Slonosky <VickyS08@users.noreply.github.com>, Hervé Hacot <hhacot@users.noreply.github.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/open-data-rescue/csv2sef
8
+ Project-URL: Issues, https://github.com/open-data-rescue/csv2sef/issues
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Operating System :: OS Independent
11
+ Requires-Python: >=3.14
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: cdrutilities>=1.0.1
15
+ Dynamic: license-file
16
+
17
+ # csv2sef
18
+ csv2sef - convert transcribed historical weather observations (csv / Excel)
19
+ into the Station Exchange Format (SEF 1.0.0).
20
+
21
+ Authors: Hervé Hacot, Victoria Slonosky (Open Data Rescue)
22
+
23
+ Please cite:
24
+ Slonosky, V., Black, R., Podolsky, L., Wang, X. & Cheng, V. (2026).
25
+ Transcribing historical Canadian weather data. Scientific Data 13, 678.
26
+ https://doi.org/10.1038/s41597-025-06036-y
27
+
28
+ ## Installation
29
+
30
+ If using this repository, ensure that you have uv installed
31
+
32
+ 1. clone the repo
33
+ 2. run `uv sync`
34
+ 3. build using `uv build`
35
+ 4. run csv2sef using
36
+ ```
37
+ uv run csv2sef
38
+ ```
39
+
40
+ ## Development
41
+
@@ -0,0 +1,22 @@
1
+ csv2sef/__init__.py,sha256=EJ98lpcoRamkDRkRHfp-MU7e4c-8yv8GrLuKTwwoEYU,48
2
+ csv2sef/checks.py,sha256=vkD44dhRDRWanbg6dCtpkC7XY3kdwbIRUfzZkgGYiSU,3487
3
+ csv2sef/cli.py,sha256=APtvKGTWlux1bJ0ulgDX73og_34l31jUBWt7neGvzX4,4803
4
+ csv2sef/convert.py,sha256=rxHLRQBjwxyOSZpX5zFebCmik_ZjQYVZa4mfufhQ0rE,22750
5
+ csv2sef/headers.py,sha256=onkG6-n18yc8UWYl97AYlrcVf95q3y2puard0WkORfk,3121
6
+ csv2sef/metadata.py,sha256=ferpG7ZK_rnBTwn41gJY9lJG5ZBpZcjcJ6ekEj70pio,10481
7
+ csv2sef/pipeline.py,sha256=fZQ9QTHRPMsXWt_hJRBtuLTpObbFnQNWfe1qlTU4gi8,23404
8
+ csv2sef/reader.py,sha256=u_h6ZS3WwaKRA_KoeloRej-Nj5GQ3bHNFYDRp_v7xYo,4826
9
+ csv2sef/sef.py,sha256=DlNMzx6ezB-kDw3uh4b6dFXKgXuozbQ3pxms477W3Hs,4958
10
+ csv2sef/tables.py,sha256=dlFLG4lyYGu1atcoHkIVM0T2kQN-JjP-SR4XlS2j8bk,3162
11
+ csv2sef/data/ranges.csv,sha256=E0zDE1K_9SAiJIS-xtkN2LAVHquBIPCyWBV5i6ZilBg,767
12
+ csv2sef/data/units.csv,sha256=thKjmiaqX4Ccfum59hcph4XiKGNtSOarQ0ophXhPhOU,3679
13
+ csv2sef/data/variable_aliases.csv,sha256=QfL7rzucA8xsBEIknd6-vGb_fgXOKkDnKYoI_EkbDdw,1531
14
+ csv2sef/data/variables.csv,sha256=pZQySUA-ny6T15pcgjMwuNb8zr3wGJAVoFhhCxDKMjI,3947
15
+ csv2sef/data/weather_terms.csv,sha256=p1PIjRCEkRdPvFkqPTVg1bTDTOR4g5xoVJfyMb-HIlE,1947
16
+ csv2sef/data/wind_words.csv,sha256=BN1nOgcF8SQy0GiMRtAFLg5TN2FLlH0BOiPHO9CMmYc,803
17
+ csv2sef-1.0.0.dist-info/licenses/LICENSE,sha256=Mt84dPHtoOMyVhl5oJiOvohG9rDiZ_7X-112mI-fK3A,1073
18
+ csv2sef-1.0.0.dist-info/METADATA,sha256=IdCClE6k-OJT5Cl9M8k2h5saEMiJkuPPTba2HmjG6qo,1290
19
+ csv2sef-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
20
+ csv2sef-1.0.0.dist-info/entry_points.txt,sha256=CLmUf9PYkxl-S-8N0DUXXhJuaWdlNGaUtdrauP1ATUE,41
21
+ csv2sef-1.0.0.dist-info/top_level.txt,sha256=q1K9t3YVyR67gXAjjjGG_torjyG7D8YmRTBbmH03jEc,8
22
+ csv2sef-1.0.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ csv2sef = csv2sef:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Open Data Rescue
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ csv2sef