csv2sef 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
csv2sef/data/units.csv ADDED
@@ -0,0 +1,91 @@
1
+ alias,unit,description
2
+ f,F,degrees Fahrenheit
3
+ °f,F,degrees Fahrenheit
4
+ ºf,F,degrees Fahrenheit
5
+ degf,F,degrees Fahrenheit
6
+ fahrenheit,F,degrees Fahrenheit
7
+ c,C,degrees Celsius
8
+ °c,C,degrees Celsius
9
+ ºc,C,degrees Celsius
10
+ degc,C,degrees Celsius
11
+ celsius,C,degrees Celsius
12
+ centigrade,C,degrees Celsius
13
+ r,R,degrees Reaumur
14
+ °r,R,degrees Reaumur
15
+ reaumur,R,degrees Reaumur
16
+ réaumur,R,degrees Reaumur
17
+ fdiff,Fdiff,temperature difference in Fahrenheit degrees
18
+ cdiff,Cdiff,temperature difference in Celsius degrees
19
+ inhg,inHg,English inches of mercury
20
+ in hg,inHg,English inches of mercury
21
+ inches hg,inHg,English inches of mercury
22
+ mmhg,mmHg,millimetres of mercury
23
+ mm hg,mmHg,millimetres of mercury
24
+ hpa,hPa,hectopascals
25
+ mb,hPa,millibars
26
+ mbar,hPa,millibars
27
+ parisin,ParisIn,French (Paris) inches of mercury; '27 11.5' = 27 inches 11.5 lines
28
+ paris in,ParisIn,French (Paris) inches of mercury
29
+ pouces,ParisIn,French (Paris) inches of mercury
30
+ parisline,ParisLine,French (Paris) lines of mercury
31
+ lignes,ParisLine,French (Paris) lines of mercury
32
+ in,in,inches (precipitation snow evaporation)
33
+ inches,in,inches (precipitation snow evaporation)
34
+ mm,mm,millimetres (precipitation)
35
+ cm,cm,centimetres (snow)
36
+ mi,mi,miles (wind run)
37
+ miles,mi,miles (wind run)
38
+ %,%,percent
39
+ p,%,percent
40
+ pct,%,percent
41
+ percent,%,percent
42
+ rh,%,percent
43
+ sm,Sm,Smithsonian wind force scale 0-10
44
+ smithsonian,Sm,Smithsonian wind force scale 0-10
45
+ bf,Bf,Beaufort force 0-12
46
+ beaufort,Bf,Beaufort force 0-12
47
+ words,words,descriptive wind force (looked up in wind_words.csv)
48
+ bf words,words,descriptive wind force (looked up in wind_words.csv)
49
+ bf_text,words,descriptive wind force (looked up in wind_words.csv)
50
+ wtext,words,descriptive wind force (looked up in wind_words.csv)
51
+ mph,mph,miles per hour
52
+ kt,kt,knots
53
+ kts,kt,knots
54
+ knots,kt,knots
55
+ m/s,m/s,metres per second
56
+ mps,m/s,metres per second
57
+ km/h,km/h,kilometres per hour
58
+ kmh,km/h,kilometres per hour
59
+ lbsft,lbsft,wind pressure in pounds per square foot ('3 15' = 3 lb 15 oz)
60
+ lb/ft2,lbsft,wind pressure in pounds per square foot
61
+ tenths,tenths,cloud cover in tenths of sky covered
62
+ tenth,tenths,cloud cover in tenths of sky covered
63
+ tenths_clear,tenths_clear,tenths of sky CLEAR (clearness of sky)
64
+ okta,okta,cloud cover in oktas
65
+ oktas,okta,cloud cover in oktas
66
+ dir,dir,compass direction (N NNE ... or degrees)
67
+ direction,dir,compass direction (N NNE ... or degrees)
68
+ deg,dir,compass direction (N NNE ... or degrees)
69
+ lct,lct,lower or general cloud type name
70
+ uct,uct,upper cloud type name
71
+ cloudvel,cloudvel,cloud velocity scale 0-10
72
+ scale,cloudvel,scale 0-10
73
+ mno,mno,weather description (coded with weather_terms.csv)
74
+ weather,mno,weather description (coded with weather_terms.csv)
75
+ text,text,free text kept verbatim
76
+ hr,text,time of day kept as text
77
+ hh:mm,text,time of day kept as text
78
+ oz,oz,ozone scale (no conversion)
79
+ beaufort text,words,descriptive wind force (looked up in wind_words.csv)
80
+ beaufort words,words,descriptive wind force (looked up in wind_words.csv)
81
+ bf text,words,descriptive wind force (looked up in wind_words.csv)
82
+ descriptive,words,descriptive wind force (looked up in wind_words.csv)
83
+ compass,dir,compass direction (N NNE ... or degrees)
84
+ free text,text,free text kept verbatim
85
+ 16-point compass,dir,compass direction (N NNE ... or degrees)
86
+ 32-point compass,dir,compass direction (N NbE ... or degrees)
87
+ de,De,degrees Delisle (0 = boiling, 150 = freezing): C = 100 - De x 2/3
88
+ delisle,De,degrees Delisle (0 = boiling, 150 = freezing): C = 100 - De x 2/3
89
+ °de,De,degrees Delisle (0 = boiling, 150 = freezing): C = 100 - De x 2/3
90
+ de_frz,De_frz,Delisle degrees counted from the freezing point: C = x * 2/3
91
+ delisle from freezing,De_frz,Delisle degrees counted from the freezing point: C = x * 2/3
@@ -0,0 +1,73 @@
1
+ alias,code,note
2
+ # Descriptive column / field names -> variable codes. Matching ignores case, underscores and
3
+ # quotes; the longest alias at the start of a header wins (the rest is read as the time).
4
+ air temperature,ta,
5
+ air temp,ta,
6
+ temperature,ta,
7
+ thermometer air,ta,
8
+ thermometer open air,ta,
9
+ open air thermometer,ta,
10
+ thermometer,ta,
11
+ attached thermometer,atb,
12
+ barometer attached thermometer,atb,
13
+ barometer,p,
14
+ air pressure,p,
15
+ pressure,p,
16
+ sea level pressure,mslp,
17
+ air pressure at sea level,mslp,
18
+ maximum thermometer,Tx,
19
+ maximum temperature,Tx,
20
+ max temperature,Tx,
21
+ max temp,Tx,
22
+ daily maximum air temperature,Tx,
23
+ minimum thermometer,Tn,
24
+ minimum temperature,Tn,
25
+ min temperature,Tn,
26
+ min temp,Tn,
27
+ daily minimum air temperature,Tn,
28
+ grass minimum thermometer,Tng,
29
+ dry bulb,tdb,
30
+ psychrometer dry bulb,tdb,
31
+ wet bulb,tb,
32
+ psychrometer wet bulb,tb,
33
+ dew point,td,
34
+ relative humidity,rh,
35
+ humidity,rh,
36
+ vapour pressure,e,
37
+ force of vapour,e,
38
+ wind direction,dd,
39
+ wind dir,dd,
40
+ wind force,w,
41
+ wind speed,w,
42
+ wind velocity,w,
43
+ wind,w,
44
+ cloud amount,n,
45
+ cloud cover,n,
46
+ cloudiness,n,
47
+ clearness of sky,skc,
48
+ cloud type,c,
49
+ cloud kind,c,
50
+ upper cloud type,ch,
51
+ lower cloud type,cl,
52
+ cloud direction,cd,
53
+ cloud velocity,cv,
54
+ rain amount,rr,
55
+ rain,rr,
56
+ precipitation,pr,
57
+ precipitation amount,pr,
58
+ accumulated precipitation,pr,
59
+ snow depth,sd,
60
+ snow on ground,sd,
61
+ snowfall,fs,
62
+ fresh snow,fs,
63
+ snow,fs,
64
+ precipitation type,pr_type,
65
+ evaporation,eee,
66
+ weather,ww,
67
+ weather remarks,ww,
68
+ present weather,ww,
69
+ casual phenomena,ww,
70
+ remarks,remarks,
71
+ notes,remarks,
72
+ comments,remarks,
73
+ aurora,au,
@@ -0,0 +1,80 @@
1
+ code,description,unit,stat,period,quantity,sef_vbl
2
+ p,Barometer observed (station pressure),inHg,point,0,pressure,p
3
+ p_mean,Barometer daily mean,inHg,mean,day,pressure,p
4
+ p_cor,Barometer corrected,inHg,point,0,pressure,p_cor
5
+ p_cor_mean,Barometer corrected daily mean,inHg,mean,day,pressure,p_cor
6
+ pcor,Barometer corrected,inHg,point,0,pressure,pcor
7
+ pcor_mean,Barometer corrected daily mean,inHg,mean,day,pressure,pcor
8
+ mslp,Mean sea level pressure,inHg,point,0,pressure,mslp
9
+ sym,Sympiesometer,inHg,point,0,pressure,sym
10
+ atb,Barometer attached thermometer,F,point,0,temperature,atb
11
+ ats,Sympiesometer attached thermometer,F,point,0,temperature,ats
12
+ ta,Air temperature,F,point,0,temperature,ta
13
+ ta_cor,Air temperature corrected for index error,F,point,0,temperature,ta_cor
14
+ ta_mean,Air temperature daily mean,F,mean,day,temperature,ta
15
+ ta_range,Air temperature daily range,Fdiff,range,day,,ta_range
16
+ Tn,Minimum temperature,F,minimum,day,temperature,Tn
17
+ Tn_cor,Minimum temperature corrected,F,minimum,day,temperature,Tn_cor
18
+ Tx,Maximum temperature,F,maximum,day,temperature,Tx
19
+ Tx_cor,Maximum temperature corrected,F,maximum,day,temperature,Tx_cor
20
+ Tng,Grass minimum thermometer,F,minimum,day,temperature,Tng
21
+ Tgn,Grass minimum thermometer,F,minimum,day,temperature,Tng
22
+ Txs,Black bulb thermometer in sun,F,maximum,day,temperature,Txs
23
+ ts,Soil temperature,F,point,0,temperature,ts
24
+ n,Cloud cover,tenths,point,0,cloud,n
25
+ nl,Cloud amount,tenths,point,0,cloud,n
26
+ nl_mean,Cloud amount daily mean,tenths,mean,day,cloud,n
27
+ nh,Upper cloud amount,tenths,point,0,cloud,nh
28
+ hl,Lower cloud amount,tenths,point,0,cloud,nl
29
+ skc,Clearness of sky,tenths_clear,point,0,cloud,n
30
+ c,Cloud type,lct,point,0,,c
31
+ cl,Lower cloud type,lct,point,0,,cl
32
+ ch,Upper cloud type,uct,point,0,,ch
33
+ cd,Cloud direction,dir,point,0,direction,cd
34
+ hd,Upper cloud direction,dir,point,0,direction,hd
35
+ cv,Cloud velocity,cloudvel,point,0,,cv
36
+ hv,Upper cloud velocity,cloudvel,point,0,,hv
37
+ dd,Wind direction,dir,point,0,direction,dd
38
+ w,Wind force or speed,Sm,point,0,wind,w
39
+ wf,Wind force,Sm,point,0,wind,w
40
+ w_mean,Wind speed daily mean,mph,mean,day,wind,w
41
+ w_anem,Anemometer wind speed,mph,point,0,wind,w
42
+ w_anem_24,Wind run in 24 hours,mi,sum,day,,w_anem_24
43
+ fx,Maximum wind in 24 hours,mph,maximum,day,wind,fx
44
+ pr,Precipitation amount,in,sum,day,precipitation,rr
45
+ rr,Rain amount,in,sum,day,precipitation,rr
46
+ rrg,Rain on ground,in,sum,day,precipitation,rrg
47
+ sd,Snow depth,in,point,0,snow,sd
48
+ swe,Snow water equivalent,in,point,0,snow,swe
49
+ pr_type,Precipitation type,text,point,day,,pr_type
50
+ rrt,Precipitation type,text,point,day,,pr_type
51
+ ptb,Precipitation time began,hr,point,day,,ptb
52
+ pte,Precipitation time ended,hr,point,day,,pte
53
+ rtb,Rain time began,hr,point,day,,rtb
54
+ rte,Rain time ended,hr,point,day,,rte
55
+ stb,Snow time began,hr,point,day,,stb
56
+ ste,Snow time ended,hr,point,day,,ste
57
+ pr_dur,Precipitation duration,text,point,day,,pr_dur
58
+ tdb,Psychrometer dry bulb,F,point,0,temperature,tdb
59
+ tdb_cor,Dry bulb corrected,F,point,0,temperature,tdb_cor
60
+ tdb_mean,Dry bulb daily mean,F,mean,day,temperature,tdb
61
+ tb,Psychrometer wet bulb,F,point,0,temperature,tb
62
+ tb_cor,Wet bulb corrected,F,point,0,temperature,tb_cor
63
+ tb_mean,Wet bulb daily mean,F,mean,day,temperature,tb
64
+ tbn,Wet bulb minimum,F,minimum,day,temperature,tbn
65
+ tbx,Wet bulb maximum,F,maximum,day,temperature,tbx
66
+ diff,Difference wet-dry bulb,Fdiff,point,0,,diff
67
+ dep_dew,Dew point depression,Fdiff,point,0,,dep_dew
68
+ rh,Relative humidity,%,point,0,humidity,rh
69
+ rh_mean,Relative humidity daily mean,%,mean,day,humidity,rh
70
+ td,Dew point,F,point,0,temperature,td
71
+ td_range,Dew point daily range,Fdiff,range,day,,td_range
72
+ e,Vapour pressure,inHg,point,0,vapour,e
73
+ e_mean,Vapour pressure daily mean,inHg,mean,day,vapour,e
74
+ eee,Evaporation,in,sum,day,precipitation,eee
75
+ oz,Ozone,oz,point,0,,oz
76
+ ww,Weather (present),mno,point,0,,ww
77
+ w1,Weather since last observation,mno,point,0,,w1
78
+ au,Aurora,text,point,0,,au
79
+ remarks,General remarks (free text),text,point,0,,remarks
80
+ fs,Fresh snowfall,in,sum,day,snow,fs
@@ -0,0 +1,71 @@
1
+ term,code,match,language,note
2
+ # match: contains = term starts a word (rain -> rainy), word = whole word only, exact = whole cell. Rows are tried in order.
3
+ thunderstorm,TS,contains,en,
4
+ thunder,TS,contains,en,
5
+ lightning,LT,contains,en,
6
+ lightening,LT,contains,en,
7
+ freezing drizzle,FZDZ,contains,en,
8
+ freezing rain,FZRA,contains,en,
9
+ freezing fog,FZFG,contains,en,
10
+ small hail,SHGS,contains,en,
11
+ hail,GR,contains,en,
12
+ ice crystals,IC,contains,en,
13
+ ice pellets,PL,contains,en,
14
+ fog patches,BCFG,contains,en,
15
+ shallow fog,MIFG,contains,en,
16
+ blowing dust,BLDU,contains,en,
17
+ dust haze,DU,contains,en,
18
+ blowing snow,BLSN,contains,en,
19
+ drifting snow,DRSN,contains,en,
20
+ drift,DRSN,contains,en,
21
+ sleet,IP RA SN,contains,en,historical only
22
+ snow storm,SN,contains,en,
23
+ snowstorm,SN,contains,en,
24
+ drizzl,DZ,contains,en,
25
+ squall,SQ,contains,en,
26
+ shower,SH,contains,en,
27
+ snow,SN,contains,en,
28
+ rain,RA,contains,en,
29
+ fog,FG,contains,en,
30
+ mist,BR,contains,en,
31
+ haz,HZ,contains,en,
32
+ smok,FU,contains,en,
33
+ aurora,AU,contains,en,
34
+ parhelia,PARHEL,contains,en,
35
+ paraselen,PARSEL,contains,en,
36
+ halo,HALO,word,en,
37
+ corona,CORONA,contains,en,
38
+ hoar frost,FRST,contains,en,
39
+ frost,FRST,contains,en,
40
+ overcast,OVC,contains,en,
41
+ gloomy,OVC,contains,en,
42
+ dull,OVC,contains,en,
43
+ cloudy,BKN,contains,en,
44
+ clear,SKC,contains,en,
45
+ fine,SKC,word,en,
46
+ fair,SKC,word,en,
47
+ beautiful,SKC,contains,en,"beautiful (weather), as fine"
48
+ sunshine,SKC,contains,en,
49
+ sun shine,SKC,contains,en,
50
+ orage,TS,word,fr,
51
+ tonnerre,TS,contains,fr,
52
+ éclair,LT,contains,fr,
53
+ verglas,FZRA,contains,fr,
54
+ grêle,GR,contains,fr,
55
+ grele,GR,contains,fr,
56
+ poudrerie,BLSN,contains,fr,blowing snow (Quebec)
57
+ neige,SN,contains,fr,
58
+ pluie,RA,contains,fr,
59
+ bruine,DZ,contains,fr,
60
+ averse,SH,contains,fr,
61
+ brouillard,FG,contains,fr,
62
+ brume,BR,contains,fr,
63
+ gelée blanche,FRST,contains,fr,
64
+ givre,FRST,contains,fr,
65
+ aurore,AU,contains,fr,
66
+ couvert,OVC,contains,fr,
67
+ sombre,OVC,word,fr,as gloomy
68
+ nuageux,BKN,contains,fr,
69
+ serein,SKC,word,fr,
70
+ beau,SKC,word,fr,
71
+ clair,SKC,word,fr,
@@ -0,0 +1,40 @@
1
+ term,beaufort,language,note
2
+ calm,0,en,
3
+ light air,1,en,
4
+ light airs,1,en,
5
+ light breeze,2,en,
6
+ gentle breeze,3,en,
7
+ moderate breeze,4,en,
8
+ fresh breeze,5,en,
9
+ strong breeze,6,en,
10
+ moderate gale,7,en,
11
+ near gale,7,en,
12
+ fresh gale,8,en,
13
+ gale,8,en,
14
+ strong gale,9,en,
15
+ whole gale,10,en,
16
+ storm,10,en,
17
+ violent storm,11,en,
18
+ hurricane,12,en,
19
+ light,2,en,keyword fallback
20
+ gentle,3,en,keyword fallback
21
+ moderate,4,en,keyword fallback
22
+ fresh,5,en,keyword fallback
23
+ strong,6,en,keyword fallback
24
+ calme,0,fr,
25
+ point de vent,0,fr,no wind
26
+ pas de vent,0,fr,no wind
27
+ très légère brise,1,fr,
28
+ legere brise,2,fr,
29
+ légère brise,2,fr,
30
+ petite brise,3,fr,
31
+ jolie brise,4,fr,
32
+ bonne brise,5,fr,
33
+ vent frais,6,fr,
34
+ grand frais,7,fr,
35
+ coup de vent,8,fr,
36
+ fort coup de vent,9,fr,
37
+ tempête,10,fr,
38
+ tempete,10,fr,
39
+ violente tempête,11,fr,
40
+ ouragan,12,fr,
csv2sef/headers.py ADDED
@@ -0,0 +1,89 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Parsing of data-column headers.
4
+
5
+ <variable> [<time>] [[<unit>]]
6
+
7
+ ta 07:00 [F] air temperature at 07:00, degrees Fahrenheit
8
+ p 7am (inHg) barometer at 07:00, inches of mercury
9
+ Tn [F] daily (overnight) minimum
10
+ ww daily daily weather remarks
11
+ Thermometer Air AM descriptive name (variable_aliases.csv) + time word
12
+
13
+ The variable may be a code from variables.csv or a descriptive name listed in
14
+ variable_aliases.csv (or added in the metadata with alias.<name> = <code>).
15
+
16
+ Times: HH:MM, H:MM, HHMM, 7am / 2 pm, sunrise (06:00), noon / meridian (12:00),
17
+ midnight (00:00). 'daily', 'day', '24h', 'mean', 'total' or no time = daily value.
18
+ Words such as AM, PM, morning, evening, night are kept as an approximate time label:
19
+ the SEF Hour/Minute are then NA unless an exact time is given in the metadata
20
+ (column.<header> = time=HH:MM).
21
+ """
22
+ import re
23
+
24
+ NAMED_TIMES = {"sunrise": (6, 0), "noon": (12, 0), "meridian": (12, 0), "midnight": (0, 0)}
25
+ DAILY_WORDS = {"daily", "day", "24h", "24hr", "24hrs", "mean", "total", "overnight"}
26
+ TIME_WORDS = {"am", "a.m.", "pm", "p.m.", "morning", "forenoon", "afternoon", "evening",
27
+ "night", "sunset", "matin", "soir", "midi", "après-midi", "apres-midi", "nuit"}
28
+
29
+
30
+ def normalise(h):
31
+ """Collapse spaces, drop BOM / non-breaking spaces and surrounding quotes."""
32
+ h = re.sub(r"\s+", " ", (h or "").replace("", "").replace("\xa0", " ")).strip()
33
+ while len(h) >= 2 and h[0] == h[-1] and h[0] in "'\"":
34
+ h = h[1:-1].strip()
35
+ return h
36
+
37
+
38
+ def split_unit(h):
39
+ """'ta 07:00 [F]' -> ('ta 07:00', 'F')"""
40
+ h = normalise(h)
41
+ m = re.search(r"[\[\(]([^\]\)]*)[\]\)]\s*$", h)
42
+ if m:
43
+ return h[:m.start()].strip(), (m.group(1).strip() or None)
44
+ return h, None
45
+
46
+
47
+ def parse_time(tok):
48
+ t = tok.strip().lower().replace(" ", "").replace(".", "")
49
+ if t in NAMED_TIMES:
50
+ return NAMED_TIMES[t]
51
+ m = re.fullmatch(r"(\d{1,2})(?::?(\d{2}))?(?::\d{2})?(am|pm|h)?", t)
52
+ if not m:
53
+ return None
54
+ h, mi, ap = int(m.group(1)), int(m.group(2) or 0), m.group(3)
55
+ if ap == "pm" and h < 12:
56
+ h += 12
57
+ if ap == "am" and h == 12:
58
+ h = 0
59
+ if h > 24 or mi > 59:
60
+ return None
61
+ return h, mi
62
+
63
+
64
+ def parse_when(rest):
65
+ """Text after the variable -> (time, label, problem).
66
+ time None and label None = daily value; label = approximate time word."""
67
+ tok = (rest or "").strip()
68
+ if not tok or tok.lower() in DAILY_WORDS:
69
+ return None, None, None
70
+ t = parse_time(tok)
71
+ if t is not None:
72
+ return t, None, None
73
+ if tok.lower() in TIME_WORDS:
74
+ return None, tok, None
75
+ return None, None, f"could not read time '{tok}'"
76
+
77
+
78
+ class Header:
79
+ """Simple parse of a coded header: first word = variable code."""
80
+
81
+ def __init__(self, raw):
82
+ self.raw = raw
83
+ text, self.unit_token = split_unit(raw)
84
+ self.var, _, rest = text.partition(" ")
85
+ self.time, self.label, self.problem = parse_when(rest)
86
+
87
+
88
+ def parse_header(raw):
89
+ return Header(raw)
csv2sef/metadata.py ADDED
@@ -0,0 +1,245 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Station metadata and settings.
4
+
5
+ Preferred format: a two-column csv (key,value) that can be edited in a spreadsheet.
6
+ Create one with `csv2sef template`. JSON files (new or legacy config_<station>.json)
7
+ are also accepted.
8
+
9
+ Site keys name, id, lat, lon, alt, source, link, project, country, observer,
10
+ utc_offset (hours, local time minus UTC, e.g. -5; blank = nearest
11
+ whole-hour time zone from the longitude), data_policy, notes
12
+ Settings convert_to_utc (yes/no), daily_timestamp (next_midnight/date),
13
+ keep_na (yes/no), range_region (global/canada), trace_mm,
14
+ tn_night_after (yes/no), sheet, wind_words_file, weather_terms_file
15
+ Per variable unit.<code> e.g. unit.w = words
16
+ variable.<code> e.g. variable.p = header_meta=PTC=N|PGC=N
17
+ column.<header> e.g. column.ww 00:00 = period=day
18
+ unit_change e.g. w; mph; 1859-10-01; 1859-11-30 (repeatable)
19
+ alias.<name> e.g. alias.Thermometer Air = ta (descriptive header name)
20
+ ignore.<header> e.g. ignore.Observer_ID = yes
21
+ Notation caves_temperature_c (default 12.8: 'co' = Caves de l'Observatoire, Paris),
22
+ below_scale_c (e.g. -34: value given to 'mercury in bulb'; '<=N' read in C),
23
+ unmapped_weather_to_remarks (yes/no, default yes)
24
+ Long layout midnight_means_daily (yes/no): a 00:00 time in a date-time column marks a
25
+ daily value (as in database exports)
26
+ """
27
+ import csv
28
+ import json
29
+ import os
30
+ import re
31
+
32
+ TEMPLATE = [
33
+ ("# csv2sef station metadata", ""),
34
+ ("# Site", ""),
35
+ ("name", ""),
36
+ ("id", ""),
37
+ ("lat", ""),
38
+ ("lon", ""),
39
+ ("alt", ""),
40
+ ("source", ""),
41
+ ("link", ""),
42
+ ("project", ""),
43
+ ("country", ""),
44
+ ("observer", ""),
45
+ ("utc_offset", ""),
46
+ ("data_policy", "CC-BY 4.0"),
47
+ ("notes", ""),
48
+ ("# Settings (defaults shown)", ""),
49
+ ("convert_to_utc", "yes"),
50
+ ("daily_timestamp", "next_midnight"),
51
+ ("keep_na", "yes"),
52
+ ("range_region", "global"),
53
+ ("trace_mm", "0.2"),
54
+ ("midnight_means_daily", "no"),
55
+ ("unmapped_weather_to_remarks", "yes"),
56
+ ("caves_temperature_c", "12.8"),
57
+ ("below_scale_c", ""),
58
+ ("tn_night_after", "no"),
59
+ ("sheet", ""),
60
+ ("wind_words_file", ""),
61
+ ("weather_terms_file", ""),
62
+ ("# Optional: alias.<column name> = <variable code>; ignore.<column> = yes; column.<header> = time=HH:MM", ""),
63
+ ("# Units of variables whose header has no [unit]: unit.<variable>", ""),
64
+ ]
65
+
66
+ DEFAULT_SETTINGS = {
67
+ "convert_to_utc": "yes", "daily_timestamp": "next_midnight", "keep_na": "yes",
68
+ "range_region": "global", "trace_mm": "0.2", "tn_night_after": "no",
69
+ }
70
+ SITE_KEYS = {"name", "id", "lat", "lon", "alt", "source", "link", "project", "country",
71
+ "observer", "utc_offset", "data_policy", "notes", "org", "timezone"}
72
+
73
+
74
+ def truthy(v):
75
+ return str(v).strip().lower() in ("yes", "y", "true", "1")
76
+
77
+
78
+ def _parse_kv(value):
79
+ """'stat=mean; period=day' -> dict"""
80
+ out = {}
81
+ for item in value.split(";"):
82
+ if "=" in item:
83
+ k, v = item.split("=", 1)
84
+ out[k.strip()] = v.strip()
85
+ return out
86
+
87
+
88
+ class Metadata:
89
+ def __init__(self):
90
+ self.site = {}
91
+ self.settings = dict(DEFAULT_SETTINGS)
92
+ self.units = {} # variable -> unit token
93
+ self.variables = {} # variable -> overrides
94
+ self.columns = {} # header -> overrides
95
+ self.unit_changes = [] # dicts: var, unit, from, to
96
+ self.aliases = {} # descriptive column name -> variable code
97
+ self.ignore = set() # column headers to ignore
98
+ self.wind_words = {} # inline (JSON) wind words
99
+ self.weather_terms = [] # inline (JSON) weather terms
100
+ self.base_dir = "."
101
+ self.notes = [] # messages about how the file was read
102
+
103
+ # ------------------------------------------------------------------
104
+ def get(self, key, default=None):
105
+ v = self.settings.get(key, default)
106
+ return default if v in ("", None) else v
107
+
108
+ def file_setting(self, key):
109
+ v = self.get(key)
110
+ if not v:
111
+ return []
112
+ return [p if os.path.isabs(p) else os.path.join(self.base_dir, p)
113
+ for p in (s.strip() for s in str(v).split(";")) if p]
114
+
115
+ # ------------------------------------------------------------------
116
+ def finalise(self):
117
+ s = self.site
118
+ missing = [k for k in ("name", "lat", "lon") if not str(s.get(k, "")).strip()]
119
+ if missing:
120
+ raise ValueError(f"Metadata is missing required site fields: {', '.join(missing)}")
121
+ s["lat"] = float(s["lat"])
122
+ lon = float(s["lon"])
123
+ if lon > 180:
124
+ lon -= 360
125
+ self.notes.append("Longitude given as 0-360 east; converted to -180..180")
126
+ s["lon"] = round(lon, 4)
127
+ s["alt"] = s.get("alt") if str(s.get("alt", "")).strip() else "NA"
128
+ if not str(s.get("id", "")).strip():
129
+ s["id"] = re.sub(r"[^A-Za-z0-9._-]", "", s["name"].title().replace(" ", ""))
130
+ self.notes.append(f"No station id given; using '{s['id']}'")
131
+ else:
132
+ s["id"] = re.sub(r"[^A-Za-z0-9._-]", "", str(s["id"]))
133
+ s.setdefault("source", "")
134
+ s.setdefault("link", "")
135
+ if str(s.get("utc_offset", "")).strip() == "":
136
+ # nearest whole-hour time zone (negative west of Greenwich)
137
+ s["utc_offset"] = float(round(s["lon"] / 15.0))
138
+ s["utc_offset_note"] = "nearest time zone to longitude"
139
+ self.notes.append(f"No utc_offset given; using nearest time zone {s['utc_offset']:+g} h "
140
+ f"(longitude / 15 = {s['lon'] / 15:+.2f}, rounded)")
141
+ else:
142
+ s["utc_offset"] = float(s["utc_offset"])
143
+ return self
144
+
145
+
146
+ # ----------------------------------------------------------------------
147
+ def load(path):
148
+ md = Metadata()
149
+ md.base_dir = os.path.dirname(os.path.abspath(path))
150
+ if path.lower().endswith(".json"):
151
+ _load_json(path, md)
152
+ else:
153
+ _load_csv(path, md)
154
+ return md.finalise()
155
+
156
+
157
+ def _load_csv(path, md):
158
+ with open(path, encoding="utf-8-sig", newline="") as f:
159
+ for row in csv.reader(f):
160
+ if not row or not row[0].strip() or row[0].lstrip().startswith("#"):
161
+ continue
162
+ key = row[0].strip()
163
+ if key.lower() == "key":
164
+ continue # header row
165
+ value = row[1].strip() if len(row) > 1 else "" # 3rd column = comment
166
+ kl = key.lower()
167
+ if kl in SITE_KEYS:
168
+ md.site[kl] = value
169
+ elif kl.startswith("unit."):
170
+ md.units[key[5:].strip()] = value
171
+ elif kl.startswith("variable."):
172
+ md.variables[key[9:].strip()] = _parse_kv(value)
173
+ elif kl.startswith("alias."):
174
+ md.aliases[key[6:].strip()] = value
175
+ elif kl.startswith("ignore."):
176
+ if truthy(value or "yes"):
177
+ md.ignore.add(" ".join(key[7:].split()).lower())
178
+ elif kl.startswith("column."):
179
+ md.columns[" ".join(key[7:].split())] = _parse_kv(value)
180
+ elif kl == "unit_change":
181
+ parts = [p.strip() for p in value.split(";")]
182
+ if len(parts) >= 2:
183
+ md.unit_changes.append({"var": parts[0], "unit": parts[1],
184
+ "from": parts[2] if len(parts) > 2 else "",
185
+ "to": parts[3] if len(parts) > 3 else ""})
186
+ else:
187
+ md.settings[kl] = value
188
+
189
+
190
+ def _load_json(path, md):
191
+ with open(path, encoding="utf-8") as f:
192
+ j = json.load(f)
193
+ site = {k: v for k, v in j.get("site", {}).items() if not k.startswith("_")}
194
+ legacy = "fields" in j or "types" in j
195
+ # utcOffset in the legacy configs = hours ADDED to local time to get UTC (Halifax: 4)
196
+ if "utcOffset" in site and "utc_offset" not in site:
197
+ off = float(site.pop("utcOffset"))
198
+ try:
199
+ lon = float(site.get("lon"))
200
+ lon = lon - 360 if lon > 180 else lon
201
+ except (TypeError, ValueError):
202
+ lon = 0
203
+ # west of Greenwich local time is behind UTC: a positive legacy value means "add"
204
+ if (lon < 0 and off > 0) or (lon > 0 and off < 0):
205
+ off = -off
206
+ site["utc_offset"] = off
207
+ md.notes.append(f"utcOffset read as local time minus UTC = {off:+g} h")
208
+ for k, v in site.items():
209
+ md.site[k.lower()] = v
210
+ if legacy:
211
+ md.notes.append("Legacy config file: only 'site' and 'unitOverride' are used")
212
+ for ov in j.get("unitOverride", []):
213
+ md.unit_changes.append({"var": ov["type"], "unit": ov["unit"],
214
+ "from": ov.get("fromDate", ""), "to": ov.get("toDate", "")})
215
+ proc = {k: v for k, v in j.get("processing", {}).items() if not k.startswith("_")}
216
+ for k, v in proc.items():
217
+ if isinstance(v, bool):
218
+ v = "yes" if v else "no"
219
+ md.settings[k] = str(v) if not isinstance(v, list) else v
220
+ if "sef_meta" in site:
221
+ md.site["notes"] = site["sef_meta"]
222
+ for var, ov in j.get("variables", {}).items():
223
+ if var.startswith("_"):
224
+ continue
225
+ ov = {k: v for k, v in ov.items() if not k.startswith("_")}
226
+ if "unit" in ov:
227
+ md.units[var] = ov.pop("unit")
228
+ md.variables[var] = ov
229
+ for col, ov in j.get("columns", {}).items():
230
+ if not col.startswith("_"):
231
+ md.columns[" ".join(col.split())] = {k: v for k, v in ov.items() if not k.startswith("_")}
232
+ md.wind_words = {k.lower(): v for k, v in j.get("wind_words", {}).items() if not k.startswith("_")}
233
+ md.weather_terms = [(k.lower(), v, "exact") for k, v in j.get("weather_terms", {}).items()
234
+ if not k.startswith("_")]
235
+
236
+
237
+ def write_template(path, variables=None):
238
+ """Write a blank metadata csv. variables: list of (code, suggested unit, description)."""
239
+ with open(path, "w", encoding="utf-8", newline="") as f:
240
+ w = csv.writer(f)
241
+ w.writerow(["key", "value"])
242
+ for k, v in TEMPLATE:
243
+ w.writerow([k, v])
244
+ for code, unit, desc in (variables or []):
245
+ w.writerow([f"unit.{code}", unit, f"# {desc}"])