csv2sef 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csv2sef/__init__.py +5 -0
- csv2sef/checks.py +78 -0
- csv2sef/cli.py +126 -0
- csv2sef/convert.py +591 -0
- csv2sef/data/ranges.csv +17 -0
- csv2sef/data/units.csv +91 -0
- csv2sef/data/variable_aliases.csv +73 -0
- csv2sef/data/variables.csv +80 -0
- csv2sef/data/weather_terms.csv +71 -0
- csv2sef/data/wind_words.csv +40 -0
- csv2sef/headers.py +89 -0
- csv2sef/metadata.py +245 -0
- csv2sef/pipeline.py +485 -0
- csv2sef/reader.py +135 -0
- csv2sef/sef.py +116 -0
- csv2sef/tables.py +101 -0
- csv2sef-1.0.0.dist-info/METADATA +41 -0
- csv2sef-1.0.0.dist-info/RECORD +22 -0
- csv2sef-1.0.0.dist-info/WHEEL +5 -0
- csv2sef-1.0.0.dist-info/entry_points.txt +2 -0
- csv2sef-1.0.0.dist-info/licenses/LICENSE +21 -0
- csv2sef-1.0.0.dist-info/top_level.txt +1 -0
csv2sef/data/units.csv
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
alias,unit,description
|
|
2
|
+
f,F,degrees Fahrenheit
|
|
3
|
+
°f,F,degrees Fahrenheit
|
|
4
|
+
ºf,F,degrees Fahrenheit
|
|
5
|
+
degf,F,degrees Fahrenheit
|
|
6
|
+
fahrenheit,F,degrees Fahrenheit
|
|
7
|
+
c,C,degrees Celsius
|
|
8
|
+
°c,C,degrees Celsius
|
|
9
|
+
ºc,C,degrees Celsius
|
|
10
|
+
degc,C,degrees Celsius
|
|
11
|
+
celsius,C,degrees Celsius
|
|
12
|
+
centigrade,C,degrees Celsius
|
|
13
|
+
r,R,degrees Reaumur
|
|
14
|
+
°r,R,degrees Reaumur
|
|
15
|
+
reaumur,R,degrees Reaumur
|
|
16
|
+
réaumur,R,degrees Reaumur
|
|
17
|
+
fdiff,Fdiff,temperature difference in Fahrenheit degrees
|
|
18
|
+
cdiff,Cdiff,temperature difference in Celsius degrees
|
|
19
|
+
inhg,inHg,English inches of mercury
|
|
20
|
+
in hg,inHg,English inches of mercury
|
|
21
|
+
inches hg,inHg,English inches of mercury
|
|
22
|
+
mmhg,mmHg,millimetres of mercury
|
|
23
|
+
mm hg,mmHg,millimetres of mercury
|
|
24
|
+
hpa,hPa,hectopascals
|
|
25
|
+
mb,hPa,millibars
|
|
26
|
+
mbar,hPa,millibars
|
|
27
|
+
parisin,ParisIn,French (Paris) inches of mercury; '27 11.5' = 27 inches 11.5 lines
|
|
28
|
+
paris in,ParisIn,French (Paris) inches of mercury
|
|
29
|
+
pouces,ParisIn,French (Paris) inches of mercury
|
|
30
|
+
parisline,ParisLine,French (Paris) lines of mercury
|
|
31
|
+
lignes,ParisLine,French (Paris) lines of mercury
|
|
32
|
+
in,in,inches (precipitation snow evaporation)
|
|
33
|
+
inches,in,inches (precipitation snow evaporation)
|
|
34
|
+
mm,mm,millimetres (precipitation)
|
|
35
|
+
cm,cm,centimetres (snow)
|
|
36
|
+
mi,mi,miles (wind run)
|
|
37
|
+
miles,mi,miles (wind run)
|
|
38
|
+
%,%,percent
|
|
39
|
+
p,%,percent
|
|
40
|
+
pct,%,percent
|
|
41
|
+
percent,%,percent
|
|
42
|
+
rh,%,percent
|
|
43
|
+
sm,Sm,Smithsonian wind force scale 0-10
|
|
44
|
+
smithsonian,Sm,Smithsonian wind force scale 0-10
|
|
45
|
+
bf,Bf,Beaufort force 0-12
|
|
46
|
+
beaufort,Bf,Beaufort force 0-12
|
|
47
|
+
words,words,descriptive wind force (looked up in wind_words.csv)
|
|
48
|
+
bf words,words,descriptive wind force (looked up in wind_words.csv)
|
|
49
|
+
bf_text,words,descriptive wind force (looked up in wind_words.csv)
|
|
50
|
+
wtext,words,descriptive wind force (looked up in wind_words.csv)
|
|
51
|
+
mph,mph,miles per hour
|
|
52
|
+
kt,kt,knots
|
|
53
|
+
kts,kt,knots
|
|
54
|
+
knots,kt,knots
|
|
55
|
+
m/s,m/s,metres per second
|
|
56
|
+
mps,m/s,metres per second
|
|
57
|
+
km/h,km/h,kilometres per hour
|
|
58
|
+
kmh,km/h,kilometres per hour
|
|
59
|
+
lbsft,lbsft,wind pressure in pounds per square foot ('3 15' = 3 lb 15 oz)
|
|
60
|
+
lb/ft2,lbsft,wind pressure in pounds per square foot
|
|
61
|
+
tenths,tenths,cloud cover in tenths of sky covered
|
|
62
|
+
tenth,tenths,cloud cover in tenths of sky covered
|
|
63
|
+
tenths_clear,tenths_clear,tenths of sky CLEAR (clearness of sky)
|
|
64
|
+
okta,okta,cloud cover in oktas
|
|
65
|
+
oktas,okta,cloud cover in oktas
|
|
66
|
+
dir,dir,compass direction (N NNE ... or degrees)
|
|
67
|
+
direction,dir,compass direction (N NNE ... or degrees)
|
|
68
|
+
deg,dir,compass direction (N NNE ... or degrees)
|
|
69
|
+
lct,lct,lower or general cloud type name
|
|
70
|
+
uct,uct,upper cloud type name
|
|
71
|
+
cloudvel,cloudvel,cloud velocity scale 0-10
|
|
72
|
+
scale,cloudvel,scale 0-10
|
|
73
|
+
mno,mno,weather description (coded with weather_terms.csv)
|
|
74
|
+
weather,mno,weather description (coded with weather_terms.csv)
|
|
75
|
+
text,text,free text kept verbatim
|
|
76
|
+
hr,text,time of day kept as text
|
|
77
|
+
hh:mm,text,time of day kept as text
|
|
78
|
+
oz,oz,ozone scale (no conversion)
|
|
79
|
+
beaufort text,words,descriptive wind force (looked up in wind_words.csv)
|
|
80
|
+
beaufort words,words,descriptive wind force (looked up in wind_words.csv)
|
|
81
|
+
bf text,words,descriptive wind force (looked up in wind_words.csv)
|
|
82
|
+
descriptive,words,descriptive wind force (looked up in wind_words.csv)
|
|
83
|
+
compass,dir,compass direction (N NNE ... or degrees)
|
|
84
|
+
free text,text,free text kept verbatim
|
|
85
|
+
16-point compass,dir,compass direction (N NNE ... or degrees)
|
|
86
|
+
32-point compass,dir,compass direction (N NbE ... or degrees)
|
|
87
|
+
de,De,degrees Delisle (0 = boiling, 150 = freezing): C = 100 - De x 2/3
|
|
88
|
+
delisle,De,degrees Delisle (0 = boiling, 150 = freezing): C = 100 - De x 2/3
|
|
89
|
+
°de,De,degrees Delisle (0 = boiling, 150 = freezing): C = 100 - De x 2/3
|
|
90
|
+
de_frz,De_frz,Delisle degrees counted from the freezing point: C = x * 2/3
|
|
91
|
+
delisle from freezing,De_frz,Delisle degrees counted from the freezing point: C = x * 2/3
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
alias,code,note
|
|
2
|
+
# Descriptive column / field names -> variable codes. Matching ignores case, underscores and
|
|
3
|
+
# quotes; the longest alias at the start of a header wins (the rest is read as the time).
|
|
4
|
+
air temperature,ta,
|
|
5
|
+
air temp,ta,
|
|
6
|
+
temperature,ta,
|
|
7
|
+
thermometer air,ta,
|
|
8
|
+
thermometer open air,ta,
|
|
9
|
+
open air thermometer,ta,
|
|
10
|
+
thermometer,ta,
|
|
11
|
+
attached thermometer,atb,
|
|
12
|
+
barometer attached thermometer,atb,
|
|
13
|
+
barometer,p,
|
|
14
|
+
air pressure,p,
|
|
15
|
+
pressure,p,
|
|
16
|
+
sea level pressure,mslp,
|
|
17
|
+
air pressure at sea level,mslp,
|
|
18
|
+
maximum thermometer,Tx,
|
|
19
|
+
maximum temperature,Tx,
|
|
20
|
+
max temperature,Tx,
|
|
21
|
+
max temp,Tx,
|
|
22
|
+
daily maximum air temperature,Tx,
|
|
23
|
+
minimum thermometer,Tn,
|
|
24
|
+
minimum temperature,Tn,
|
|
25
|
+
min temperature,Tn,
|
|
26
|
+
min temp,Tn,
|
|
27
|
+
daily minimum air temperature,Tn,
|
|
28
|
+
grass minimum thermometer,Tng,
|
|
29
|
+
dry bulb,tdb,
|
|
30
|
+
psychrometer dry bulb,tdb,
|
|
31
|
+
wet bulb,tb,
|
|
32
|
+
psychrometer wet bulb,tb,
|
|
33
|
+
dew point,td,
|
|
34
|
+
relative humidity,rh,
|
|
35
|
+
humidity,rh,
|
|
36
|
+
vapour pressure,e,
|
|
37
|
+
force of vapour,e,
|
|
38
|
+
wind direction,dd,
|
|
39
|
+
wind dir,dd,
|
|
40
|
+
wind force,w,
|
|
41
|
+
wind speed,w,
|
|
42
|
+
wind velocity,w,
|
|
43
|
+
wind,w,
|
|
44
|
+
cloud amount,n,
|
|
45
|
+
cloud cover,n,
|
|
46
|
+
cloudiness,n,
|
|
47
|
+
clearness of sky,skc,
|
|
48
|
+
cloud type,c,
|
|
49
|
+
cloud kind,c,
|
|
50
|
+
upper cloud type,ch,
|
|
51
|
+
lower cloud type,cl,
|
|
52
|
+
cloud direction,cd,
|
|
53
|
+
cloud velocity,cv,
|
|
54
|
+
rain amount,rr,
|
|
55
|
+
rain,rr,
|
|
56
|
+
precipitation,pr,
|
|
57
|
+
precipitation amount,pr,
|
|
58
|
+
accumulated precipitation,pr,
|
|
59
|
+
snow depth,sd,
|
|
60
|
+
snow on ground,sd,
|
|
61
|
+
snowfall,fs,
|
|
62
|
+
fresh snow,fs,
|
|
63
|
+
snow,fs,
|
|
64
|
+
precipitation type,pr_type,
|
|
65
|
+
evaporation,eee,
|
|
66
|
+
weather,ww,
|
|
67
|
+
weather remarks,ww,
|
|
68
|
+
present weather,ww,
|
|
69
|
+
casual phenomena,ww,
|
|
70
|
+
remarks,remarks,
|
|
71
|
+
notes,remarks,
|
|
72
|
+
comments,remarks,
|
|
73
|
+
aurora,au,
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
code,description,unit,stat,period,quantity,sef_vbl
|
|
2
|
+
p,Barometer observed (station pressure),inHg,point,0,pressure,p
|
|
3
|
+
p_mean,Barometer daily mean,inHg,mean,day,pressure,p
|
|
4
|
+
p_cor,Barometer corrected,inHg,point,0,pressure,p_cor
|
|
5
|
+
p_cor_mean,Barometer corrected daily mean,inHg,mean,day,pressure,p_cor
|
|
6
|
+
pcor,Barometer corrected,inHg,point,0,pressure,pcor
|
|
7
|
+
pcor_mean,Barometer corrected daily mean,inHg,mean,day,pressure,pcor
|
|
8
|
+
mslp,Mean sea level pressure,inHg,point,0,pressure,mslp
|
|
9
|
+
sym,Sympiesometer,inHg,point,0,pressure,sym
|
|
10
|
+
atb,Barometer attached thermometer,F,point,0,temperature,atb
|
|
11
|
+
ats,Sympiesometer attached thermometer,F,point,0,temperature,ats
|
|
12
|
+
ta,Air temperature,F,point,0,temperature,ta
|
|
13
|
+
ta_cor,Air temperature corrected for index error,F,point,0,temperature,ta_cor
|
|
14
|
+
ta_mean,Air temperature daily mean,F,mean,day,temperature,ta
|
|
15
|
+
ta_range,Air temperature daily range,Fdiff,range,day,,ta_range
|
|
16
|
+
Tn,Minimum temperature,F,minimum,day,temperature,Tn
|
|
17
|
+
Tn_cor,Minimum temperature corrected,F,minimum,day,temperature,Tn_cor
|
|
18
|
+
Tx,Maximum temperature,F,maximum,day,temperature,Tx
|
|
19
|
+
Tx_cor,Maximum temperature corrected,F,maximum,day,temperature,Tx_cor
|
|
20
|
+
Tng,Grass minimum thermometer,F,minimum,day,temperature,Tng
|
|
21
|
+
Tgn,Grass minimum thermometer,F,minimum,day,temperature,Tng
|
|
22
|
+
Txs,Black bulb thermometer in sun,F,maximum,day,temperature,Txs
|
|
23
|
+
ts,Soil temperature,F,point,0,temperature,ts
|
|
24
|
+
n,Cloud cover,tenths,point,0,cloud,n
|
|
25
|
+
nl,Cloud amount,tenths,point,0,cloud,n
|
|
26
|
+
nl_mean,Cloud amount daily mean,tenths,mean,day,cloud,n
|
|
27
|
+
nh,Upper cloud amount,tenths,point,0,cloud,nh
|
|
28
|
+
hl,Lower cloud amount,tenths,point,0,cloud,nl
|
|
29
|
+
skc,Clearness of sky,tenths_clear,point,0,cloud,n
|
|
30
|
+
c,Cloud type,lct,point,0,,c
|
|
31
|
+
cl,Lower cloud type,lct,point,0,,cl
|
|
32
|
+
ch,Upper cloud type,uct,point,0,,ch
|
|
33
|
+
cd,Cloud direction,dir,point,0,direction,cd
|
|
34
|
+
hd,Upper cloud direction,dir,point,0,direction,hd
|
|
35
|
+
cv,Cloud velocity,cloudvel,point,0,,cv
|
|
36
|
+
hv,Upper cloud velocity,cloudvel,point,0,,hv
|
|
37
|
+
dd,Wind direction,dir,point,0,direction,dd
|
|
38
|
+
w,Wind force or speed,Sm,point,0,wind,w
|
|
39
|
+
wf,Wind force,Sm,point,0,wind,w
|
|
40
|
+
w_mean,Wind speed daily mean,mph,mean,day,wind,w
|
|
41
|
+
w_anem,Anemometer wind speed,mph,point,0,wind,w
|
|
42
|
+
w_anem_24,Wind run in 24 hours,mi,sum,day,,w_anem_24
|
|
43
|
+
fx,Maximum wind in 24 hours,mph,maximum,day,wind,fx
|
|
44
|
+
pr,Precipitation amount,in,sum,day,precipitation,rr
|
|
45
|
+
rr,Rain amount,in,sum,day,precipitation,rr
|
|
46
|
+
rrg,Rain on ground,in,sum,day,precipitation,rrg
|
|
47
|
+
sd,Snow depth,in,point,0,snow,sd
|
|
48
|
+
swe,Snow water equivalent,in,point,0,snow,swe
|
|
49
|
+
pr_type,Precipitation type,text,point,day,,pr_type
|
|
50
|
+
rrt,Precipitation type,text,point,day,,pr_type
|
|
51
|
+
ptb,Precipitation time began,hr,point,day,,ptb
|
|
52
|
+
pte,Precipitation time ended,hr,point,day,,pte
|
|
53
|
+
rtb,Rain time began,hr,point,day,,rtb
|
|
54
|
+
rte,Rain time ended,hr,point,day,,rte
|
|
55
|
+
stb,Snow time began,hr,point,day,,stb
|
|
56
|
+
ste,Snow time ended,hr,point,day,,ste
|
|
57
|
+
pr_dur,Precipitation duration,text,point,day,,pr_dur
|
|
58
|
+
tdb,Psychrometer dry bulb,F,point,0,temperature,tdb
|
|
59
|
+
tdb_cor,Dry bulb corrected,F,point,0,temperature,tdb_cor
|
|
60
|
+
tdb_mean,Dry bulb daily mean,F,mean,day,temperature,tdb
|
|
61
|
+
tb,Psychrometer wet bulb,F,point,0,temperature,tb
|
|
62
|
+
tb_cor,Wet bulb corrected,F,point,0,temperature,tb_cor
|
|
63
|
+
tb_mean,Wet bulb daily mean,F,mean,day,temperature,tb
|
|
64
|
+
tbn,Wet bulb minimum,F,minimum,day,temperature,tbn
|
|
65
|
+
tbx,Wet bulb maximum,F,maximum,day,temperature,tbx
|
|
66
|
+
diff,Difference wet-dry bulb,Fdiff,point,0,,diff
|
|
67
|
+
dep_dew,Dew point depression,Fdiff,point,0,,dep_dew
|
|
68
|
+
rh,Relative humidity,%,point,0,humidity,rh
|
|
69
|
+
rh_mean,Relative humidity daily mean,%,mean,day,humidity,rh
|
|
70
|
+
td,Dew point,F,point,0,temperature,td
|
|
71
|
+
td_range,Dew point daily range,Fdiff,range,day,,td_range
|
|
72
|
+
e,Vapour pressure,inHg,point,0,vapour,e
|
|
73
|
+
e_mean,Vapour pressure daily mean,inHg,mean,day,vapour,e
|
|
74
|
+
eee,Evaporation,in,sum,day,precipitation,eee
|
|
75
|
+
oz,Ozone,oz,point,0,,oz
|
|
76
|
+
ww,Weather (present),mno,point,0,,ww
|
|
77
|
+
w1,Weather since last observation,mno,point,0,,w1
|
|
78
|
+
au,Aurora,text,point,0,,au
|
|
79
|
+
remarks,General remarks (free text),text,point,0,,remarks
|
|
80
|
+
fs,Fresh snowfall,in,sum,day,snow,fs
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
term,code,match,language,note
|
|
2
|
+
# match: contains = term starts a word (rain -> rainy), word = whole word only, exact = whole cell. Rows are tried in order.
|
|
3
|
+
thunderstorm,TS,contains,en,
|
|
4
|
+
thunder,TS,contains,en,
|
|
5
|
+
lightning,LT,contains,en,
|
|
6
|
+
lightening,LT,contains,en,
|
|
7
|
+
freezing drizzle,FZDZ,contains,en,
|
|
8
|
+
freezing rain,FZRA,contains,en,
|
|
9
|
+
freezing fog,FZFG,contains,en,
|
|
10
|
+
small hail,SHGS,contains,en,
|
|
11
|
+
hail,GR,contains,en,
|
|
12
|
+
ice crystals,IC,contains,en,
|
|
13
|
+
ice pellets,PL,contains,en,
|
|
14
|
+
fog patches,BCFG,contains,en,
|
|
15
|
+
shallow fog,MIFG,contains,en,
|
|
16
|
+
blowing dust,BLDU,contains,en,
|
|
17
|
+
dust haze,DU,contains,en,
|
|
18
|
+
blowing snow,BLSN,contains,en,
|
|
19
|
+
drifting snow,DRSN,contains,en,
|
|
20
|
+
drift,DRSN,contains,en,
|
|
21
|
+
sleet,IP RA SN,contains,en,historical only
|
|
22
|
+
snow storm,SN,contains,en,
|
|
23
|
+
snowstorm,SN,contains,en,
|
|
24
|
+
drizzl,DZ,contains,en,
|
|
25
|
+
squall,SQ,contains,en,
|
|
26
|
+
shower,SH,contains,en,
|
|
27
|
+
snow,SN,contains,en,
|
|
28
|
+
rain,RA,contains,en,
|
|
29
|
+
fog,FG,contains,en,
|
|
30
|
+
mist,BR,contains,en,
|
|
31
|
+
haz,HZ,contains,en,
|
|
32
|
+
smok,FU,contains,en,
|
|
33
|
+
aurora,AU,contains,en,
|
|
34
|
+
parhelia,PARHEL,contains,en,
|
|
35
|
+
paraselen,PARSEL,contains,en,
|
|
36
|
+
halo,HALO,word,en,
|
|
37
|
+
corona,CORONA,contains,en,
|
|
38
|
+
hoar frost,FRST,contains,en,
|
|
39
|
+
frost,FRST,contains,en,
|
|
40
|
+
overcast,OVC,contains,en,
|
|
41
|
+
gloomy,OVC,contains,en,
|
|
42
|
+
dull,OVC,contains,en,
|
|
43
|
+
cloudy,BKN,contains,en,
|
|
44
|
+
clear,SKC,contains,en,
|
|
45
|
+
fine,SKC,word,en,
|
|
46
|
+
fair,SKC,word,en,
|
|
47
|
+
beautiful,SKC,contains,en,"beautiful (weather), as fine"
|
|
48
|
+
sunshine,SKC,contains,en,
|
|
49
|
+
sun shine,SKC,contains,en,
|
|
50
|
+
orage,TS,word,fr,
|
|
51
|
+
tonnerre,TS,contains,fr,
|
|
52
|
+
éclair,LT,contains,fr,
|
|
53
|
+
verglas,FZRA,contains,fr,
|
|
54
|
+
grêle,GR,contains,fr,
|
|
55
|
+
grele,GR,contains,fr,
|
|
56
|
+
poudrerie,BLSN,contains,fr,blowing snow (Quebec)
|
|
57
|
+
neige,SN,contains,fr,
|
|
58
|
+
pluie,RA,contains,fr,
|
|
59
|
+
bruine,DZ,contains,fr,
|
|
60
|
+
averse,SH,contains,fr,
|
|
61
|
+
brouillard,FG,contains,fr,
|
|
62
|
+
brume,BR,contains,fr,
|
|
63
|
+
gelée blanche,FRST,contains,fr,
|
|
64
|
+
givre,FRST,contains,fr,
|
|
65
|
+
aurore,AU,contains,fr,
|
|
66
|
+
couvert,OVC,contains,fr,
|
|
67
|
+
sombre,OVC,word,fr,as gloomy
|
|
68
|
+
nuageux,BKN,contains,fr,
|
|
69
|
+
serein,SKC,word,fr,
|
|
70
|
+
beau,SKC,word,fr,
|
|
71
|
+
clair,SKC,word,fr,
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
term,beaufort,language,note
|
|
2
|
+
calm,0,en,
|
|
3
|
+
light air,1,en,
|
|
4
|
+
light airs,1,en,
|
|
5
|
+
light breeze,2,en,
|
|
6
|
+
gentle breeze,3,en,
|
|
7
|
+
moderate breeze,4,en,
|
|
8
|
+
fresh breeze,5,en,
|
|
9
|
+
strong breeze,6,en,
|
|
10
|
+
moderate gale,7,en,
|
|
11
|
+
near gale,7,en,
|
|
12
|
+
fresh gale,8,en,
|
|
13
|
+
gale,8,en,
|
|
14
|
+
strong gale,9,en,
|
|
15
|
+
whole gale,10,en,
|
|
16
|
+
storm,10,en,
|
|
17
|
+
violent storm,11,en,
|
|
18
|
+
hurricane,12,en,
|
|
19
|
+
light,2,en,keyword fallback
|
|
20
|
+
gentle,3,en,keyword fallback
|
|
21
|
+
moderate,4,en,keyword fallback
|
|
22
|
+
fresh,5,en,keyword fallback
|
|
23
|
+
strong,6,en,keyword fallback
|
|
24
|
+
calme,0,fr,
|
|
25
|
+
point de vent,0,fr,no wind
|
|
26
|
+
pas de vent,0,fr,no wind
|
|
27
|
+
très légère brise,1,fr,
|
|
28
|
+
legere brise,2,fr,
|
|
29
|
+
légère brise,2,fr,
|
|
30
|
+
petite brise,3,fr,
|
|
31
|
+
jolie brise,4,fr,
|
|
32
|
+
bonne brise,5,fr,
|
|
33
|
+
vent frais,6,fr,
|
|
34
|
+
grand frais,7,fr,
|
|
35
|
+
coup de vent,8,fr,
|
|
36
|
+
fort coup de vent,9,fr,
|
|
37
|
+
tempête,10,fr,
|
|
38
|
+
tempete,10,fr,
|
|
39
|
+
violente tempête,11,fr,
|
|
40
|
+
ouragan,12,fr,
|
csv2sef/headers.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Parsing of data-column headers.
|
|
4
|
+
|
|
5
|
+
<variable> [<time>] [[<unit>]]
|
|
6
|
+
|
|
7
|
+
ta 07:00 [F] air temperature at 07:00, degrees Fahrenheit
|
|
8
|
+
p 7am (inHg) barometer at 07:00, inches of mercury
|
|
9
|
+
Tn [F] daily (overnight) minimum
|
|
10
|
+
ww daily daily weather remarks
|
|
11
|
+
Thermometer Air AM descriptive name (variable_aliases.csv) + time word
|
|
12
|
+
|
|
13
|
+
The variable may be a code from variables.csv or a descriptive name listed in
|
|
14
|
+
variable_aliases.csv (or added in the metadata with alias.<name> = <code>).
|
|
15
|
+
|
|
16
|
+
Times: HH:MM, H:MM, HHMM, 7am / 2 pm, sunrise (06:00), noon / meridian (12:00),
|
|
17
|
+
midnight (00:00). 'daily', 'day', '24h', 'mean', 'total' or no time = daily value.
|
|
18
|
+
Words such as AM, PM, morning, evening, night are kept as an approximate time label:
|
|
19
|
+
the SEF Hour/Minute are then NA unless an exact time is given in the metadata
|
|
20
|
+
(column.<header> = time=HH:MM).
|
|
21
|
+
"""
|
|
22
|
+
import re
|
|
23
|
+
|
|
24
|
+
NAMED_TIMES = {"sunrise": (6, 0), "noon": (12, 0), "meridian": (12, 0), "midnight": (0, 0)}
|
|
25
|
+
DAILY_WORDS = {"daily", "day", "24h", "24hr", "24hrs", "mean", "total", "overnight"}
|
|
26
|
+
TIME_WORDS = {"am", "a.m.", "pm", "p.m.", "morning", "forenoon", "afternoon", "evening",
|
|
27
|
+
"night", "sunset", "matin", "soir", "midi", "après-midi", "apres-midi", "nuit"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def normalise(h):
|
|
31
|
+
"""Collapse spaces, drop BOM / non-breaking spaces and surrounding quotes."""
|
|
32
|
+
h = re.sub(r"\s+", " ", (h or "").replace("", "").replace("\xa0", " ")).strip()
|
|
33
|
+
while len(h) >= 2 and h[0] == h[-1] and h[0] in "'\"":
|
|
34
|
+
h = h[1:-1].strip()
|
|
35
|
+
return h
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def split_unit(h):
|
|
39
|
+
"""'ta 07:00 [F]' -> ('ta 07:00', 'F')"""
|
|
40
|
+
h = normalise(h)
|
|
41
|
+
m = re.search(r"[\[\(]([^\]\)]*)[\]\)]\s*$", h)
|
|
42
|
+
if m:
|
|
43
|
+
return h[:m.start()].strip(), (m.group(1).strip() or None)
|
|
44
|
+
return h, None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def parse_time(tok):
|
|
48
|
+
t = tok.strip().lower().replace(" ", "").replace(".", "")
|
|
49
|
+
if t in NAMED_TIMES:
|
|
50
|
+
return NAMED_TIMES[t]
|
|
51
|
+
m = re.fullmatch(r"(\d{1,2})(?::?(\d{2}))?(?::\d{2})?(am|pm|h)?", t)
|
|
52
|
+
if not m:
|
|
53
|
+
return None
|
|
54
|
+
h, mi, ap = int(m.group(1)), int(m.group(2) or 0), m.group(3)
|
|
55
|
+
if ap == "pm" and h < 12:
|
|
56
|
+
h += 12
|
|
57
|
+
if ap == "am" and h == 12:
|
|
58
|
+
h = 0
|
|
59
|
+
if h > 24 or mi > 59:
|
|
60
|
+
return None
|
|
61
|
+
return h, mi
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def parse_when(rest):
|
|
65
|
+
"""Text after the variable -> (time, label, problem).
|
|
66
|
+
time None and label None = daily value; label = approximate time word."""
|
|
67
|
+
tok = (rest or "").strip()
|
|
68
|
+
if not tok or tok.lower() in DAILY_WORDS:
|
|
69
|
+
return None, None, None
|
|
70
|
+
t = parse_time(tok)
|
|
71
|
+
if t is not None:
|
|
72
|
+
return t, None, None
|
|
73
|
+
if tok.lower() in TIME_WORDS:
|
|
74
|
+
return None, tok, None
|
|
75
|
+
return None, None, f"could not read time '{tok}'"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Header:
|
|
79
|
+
"""Simple parse of a coded header: first word = variable code."""
|
|
80
|
+
|
|
81
|
+
def __init__(self, raw):
|
|
82
|
+
self.raw = raw
|
|
83
|
+
text, self.unit_token = split_unit(raw)
|
|
84
|
+
self.var, _, rest = text.partition(" ")
|
|
85
|
+
self.time, self.label, self.problem = parse_when(rest)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def parse_header(raw):
|
|
89
|
+
return Header(raw)
|
csv2sef/metadata.py
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Station metadata and settings.
|
|
4
|
+
|
|
5
|
+
Preferred format: a two-column csv (key,value) that can be edited in a spreadsheet.
|
|
6
|
+
Create one with `csv2sef template`. JSON files (new or legacy config_<station>.json)
|
|
7
|
+
are also accepted.
|
|
8
|
+
|
|
9
|
+
Site keys name, id, lat, lon, alt, source, link, project, country, observer,
|
|
10
|
+
utc_offset (hours, local time minus UTC, e.g. -5; blank = nearest
|
|
11
|
+
whole-hour time zone from the longitude), data_policy, notes
|
|
12
|
+
Settings convert_to_utc (yes/no), daily_timestamp (next_midnight/date),
|
|
13
|
+
keep_na (yes/no), range_region (global/canada), trace_mm,
|
|
14
|
+
tn_night_after (yes/no), sheet, wind_words_file, weather_terms_file
|
|
15
|
+
Per variable unit.<code> e.g. unit.w = words
|
|
16
|
+
variable.<code> e.g. variable.p = header_meta=PTC=N|PGC=N
|
|
17
|
+
column.<header> e.g. column.ww 00:00 = period=day
|
|
18
|
+
unit_change e.g. w; mph; 1859-10-01; 1859-11-30 (repeatable)
|
|
19
|
+
alias.<name> e.g. alias.Thermometer Air = ta (descriptive header name)
|
|
20
|
+
ignore.<header> e.g. ignore.Observer_ID = yes
|
|
21
|
+
Notation caves_temperature_c (default 12.8: 'co' = Caves de l'Observatoire, Paris),
|
|
22
|
+
below_scale_c (e.g. -34: value given to 'mercury in bulb'; '<=N' read in C),
|
|
23
|
+
unmapped_weather_to_remarks (yes/no, default yes)
|
|
24
|
+
Long layout midnight_means_daily (yes/no): a 00:00 time in a date-time column marks a
|
|
25
|
+
daily value (as in database exports)
|
|
26
|
+
"""
|
|
27
|
+
import csv
|
|
28
|
+
import json
|
|
29
|
+
import os
|
|
30
|
+
import re
|
|
31
|
+
|
|
32
|
+
TEMPLATE = [
|
|
33
|
+
("# csv2sef station metadata", ""),
|
|
34
|
+
("# Site", ""),
|
|
35
|
+
("name", ""),
|
|
36
|
+
("id", ""),
|
|
37
|
+
("lat", ""),
|
|
38
|
+
("lon", ""),
|
|
39
|
+
("alt", ""),
|
|
40
|
+
("source", ""),
|
|
41
|
+
("link", ""),
|
|
42
|
+
("project", ""),
|
|
43
|
+
("country", ""),
|
|
44
|
+
("observer", ""),
|
|
45
|
+
("utc_offset", ""),
|
|
46
|
+
("data_policy", "CC-BY 4.0"),
|
|
47
|
+
("notes", ""),
|
|
48
|
+
("# Settings (defaults shown)", ""),
|
|
49
|
+
("convert_to_utc", "yes"),
|
|
50
|
+
("daily_timestamp", "next_midnight"),
|
|
51
|
+
("keep_na", "yes"),
|
|
52
|
+
("range_region", "global"),
|
|
53
|
+
("trace_mm", "0.2"),
|
|
54
|
+
("midnight_means_daily", "no"),
|
|
55
|
+
("unmapped_weather_to_remarks", "yes"),
|
|
56
|
+
("caves_temperature_c", "12.8"),
|
|
57
|
+
("below_scale_c", ""),
|
|
58
|
+
("tn_night_after", "no"),
|
|
59
|
+
("sheet", ""),
|
|
60
|
+
("wind_words_file", ""),
|
|
61
|
+
("weather_terms_file", ""),
|
|
62
|
+
("# Optional: alias.<column name> = <variable code>; ignore.<column> = yes; column.<header> = time=HH:MM", ""),
|
|
63
|
+
("# Units of variables whose header has no [unit]: unit.<variable>", ""),
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
DEFAULT_SETTINGS = {
|
|
67
|
+
"convert_to_utc": "yes", "daily_timestamp": "next_midnight", "keep_na": "yes",
|
|
68
|
+
"range_region": "global", "trace_mm": "0.2", "tn_night_after": "no",
|
|
69
|
+
}
|
|
70
|
+
SITE_KEYS = {"name", "id", "lat", "lon", "alt", "source", "link", "project", "country",
|
|
71
|
+
"observer", "utc_offset", "data_policy", "notes", "org", "timezone"}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def truthy(v):
|
|
75
|
+
return str(v).strip().lower() in ("yes", "y", "true", "1")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _parse_kv(value):
|
|
79
|
+
"""'stat=mean; period=day' -> dict"""
|
|
80
|
+
out = {}
|
|
81
|
+
for item in value.split(";"):
|
|
82
|
+
if "=" in item:
|
|
83
|
+
k, v = item.split("=", 1)
|
|
84
|
+
out[k.strip()] = v.strip()
|
|
85
|
+
return out
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class Metadata:
|
|
89
|
+
def __init__(self):
|
|
90
|
+
self.site = {}
|
|
91
|
+
self.settings = dict(DEFAULT_SETTINGS)
|
|
92
|
+
self.units = {} # variable -> unit token
|
|
93
|
+
self.variables = {} # variable -> overrides
|
|
94
|
+
self.columns = {} # header -> overrides
|
|
95
|
+
self.unit_changes = [] # dicts: var, unit, from, to
|
|
96
|
+
self.aliases = {} # descriptive column name -> variable code
|
|
97
|
+
self.ignore = set() # column headers to ignore
|
|
98
|
+
self.wind_words = {} # inline (JSON) wind words
|
|
99
|
+
self.weather_terms = [] # inline (JSON) weather terms
|
|
100
|
+
self.base_dir = "."
|
|
101
|
+
self.notes = [] # messages about how the file was read
|
|
102
|
+
|
|
103
|
+
# ------------------------------------------------------------------
|
|
104
|
+
def get(self, key, default=None):
|
|
105
|
+
v = self.settings.get(key, default)
|
|
106
|
+
return default if v in ("", None) else v
|
|
107
|
+
|
|
108
|
+
def file_setting(self, key):
|
|
109
|
+
v = self.get(key)
|
|
110
|
+
if not v:
|
|
111
|
+
return []
|
|
112
|
+
return [p if os.path.isabs(p) else os.path.join(self.base_dir, p)
|
|
113
|
+
for p in (s.strip() for s in str(v).split(";")) if p]
|
|
114
|
+
|
|
115
|
+
# ------------------------------------------------------------------
|
|
116
|
+
def finalise(self):
|
|
117
|
+
s = self.site
|
|
118
|
+
missing = [k for k in ("name", "lat", "lon") if not str(s.get(k, "")).strip()]
|
|
119
|
+
if missing:
|
|
120
|
+
raise ValueError(f"Metadata is missing required site fields: {', '.join(missing)}")
|
|
121
|
+
s["lat"] = float(s["lat"])
|
|
122
|
+
lon = float(s["lon"])
|
|
123
|
+
if lon > 180:
|
|
124
|
+
lon -= 360
|
|
125
|
+
self.notes.append("Longitude given as 0-360 east; converted to -180..180")
|
|
126
|
+
s["lon"] = round(lon, 4)
|
|
127
|
+
s["alt"] = s.get("alt") if str(s.get("alt", "")).strip() else "NA"
|
|
128
|
+
if not str(s.get("id", "")).strip():
|
|
129
|
+
s["id"] = re.sub(r"[^A-Za-z0-9._-]", "", s["name"].title().replace(" ", ""))
|
|
130
|
+
self.notes.append(f"No station id given; using '{s['id']}'")
|
|
131
|
+
else:
|
|
132
|
+
s["id"] = re.sub(r"[^A-Za-z0-9._-]", "", str(s["id"]))
|
|
133
|
+
s.setdefault("source", "")
|
|
134
|
+
s.setdefault("link", "")
|
|
135
|
+
if str(s.get("utc_offset", "")).strip() == "":
|
|
136
|
+
# nearest whole-hour time zone (negative west of Greenwich)
|
|
137
|
+
s["utc_offset"] = float(round(s["lon"] / 15.0))
|
|
138
|
+
s["utc_offset_note"] = "nearest time zone to longitude"
|
|
139
|
+
self.notes.append(f"No utc_offset given; using nearest time zone {s['utc_offset']:+g} h "
|
|
140
|
+
f"(longitude / 15 = {s['lon'] / 15:+.2f}, rounded)")
|
|
141
|
+
else:
|
|
142
|
+
s["utc_offset"] = float(s["utc_offset"])
|
|
143
|
+
return self
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
# ----------------------------------------------------------------------
|
|
147
|
+
def load(path):
|
|
148
|
+
md = Metadata()
|
|
149
|
+
md.base_dir = os.path.dirname(os.path.abspath(path))
|
|
150
|
+
if path.lower().endswith(".json"):
|
|
151
|
+
_load_json(path, md)
|
|
152
|
+
else:
|
|
153
|
+
_load_csv(path, md)
|
|
154
|
+
return md.finalise()
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _load_csv(path, md):
|
|
158
|
+
with open(path, encoding="utf-8-sig", newline="") as f:
|
|
159
|
+
for row in csv.reader(f):
|
|
160
|
+
if not row or not row[0].strip() or row[0].lstrip().startswith("#"):
|
|
161
|
+
continue
|
|
162
|
+
key = row[0].strip()
|
|
163
|
+
if key.lower() == "key":
|
|
164
|
+
continue # header row
|
|
165
|
+
value = row[1].strip() if len(row) > 1 else "" # 3rd column = comment
|
|
166
|
+
kl = key.lower()
|
|
167
|
+
if kl in SITE_KEYS:
|
|
168
|
+
md.site[kl] = value
|
|
169
|
+
elif kl.startswith("unit."):
|
|
170
|
+
md.units[key[5:].strip()] = value
|
|
171
|
+
elif kl.startswith("variable."):
|
|
172
|
+
md.variables[key[9:].strip()] = _parse_kv(value)
|
|
173
|
+
elif kl.startswith("alias."):
|
|
174
|
+
md.aliases[key[6:].strip()] = value
|
|
175
|
+
elif kl.startswith("ignore."):
|
|
176
|
+
if truthy(value or "yes"):
|
|
177
|
+
md.ignore.add(" ".join(key[7:].split()).lower())
|
|
178
|
+
elif kl.startswith("column."):
|
|
179
|
+
md.columns[" ".join(key[7:].split())] = _parse_kv(value)
|
|
180
|
+
elif kl == "unit_change":
|
|
181
|
+
parts = [p.strip() for p in value.split(";")]
|
|
182
|
+
if len(parts) >= 2:
|
|
183
|
+
md.unit_changes.append({"var": parts[0], "unit": parts[1],
|
|
184
|
+
"from": parts[2] if len(parts) > 2 else "",
|
|
185
|
+
"to": parts[3] if len(parts) > 3 else ""})
|
|
186
|
+
else:
|
|
187
|
+
md.settings[kl] = value
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _load_json(path, md):
|
|
191
|
+
with open(path, encoding="utf-8") as f:
|
|
192
|
+
j = json.load(f)
|
|
193
|
+
site = {k: v for k, v in j.get("site", {}).items() if not k.startswith("_")}
|
|
194
|
+
legacy = "fields" in j or "types" in j
|
|
195
|
+
# utcOffset in the legacy configs = hours ADDED to local time to get UTC (Halifax: 4)
|
|
196
|
+
if "utcOffset" in site and "utc_offset" not in site:
|
|
197
|
+
off = float(site.pop("utcOffset"))
|
|
198
|
+
try:
|
|
199
|
+
lon = float(site.get("lon"))
|
|
200
|
+
lon = lon - 360 if lon > 180 else lon
|
|
201
|
+
except (TypeError, ValueError):
|
|
202
|
+
lon = 0
|
|
203
|
+
# west of Greenwich local time is behind UTC: a positive legacy value means "add"
|
|
204
|
+
if (lon < 0 and off > 0) or (lon > 0 and off < 0):
|
|
205
|
+
off = -off
|
|
206
|
+
site["utc_offset"] = off
|
|
207
|
+
md.notes.append(f"utcOffset read as local time minus UTC = {off:+g} h")
|
|
208
|
+
for k, v in site.items():
|
|
209
|
+
md.site[k.lower()] = v
|
|
210
|
+
if legacy:
|
|
211
|
+
md.notes.append("Legacy config file: only 'site' and 'unitOverride' are used")
|
|
212
|
+
for ov in j.get("unitOverride", []):
|
|
213
|
+
md.unit_changes.append({"var": ov["type"], "unit": ov["unit"],
|
|
214
|
+
"from": ov.get("fromDate", ""), "to": ov.get("toDate", "")})
|
|
215
|
+
proc = {k: v for k, v in j.get("processing", {}).items() if not k.startswith("_")}
|
|
216
|
+
for k, v in proc.items():
|
|
217
|
+
if isinstance(v, bool):
|
|
218
|
+
v = "yes" if v else "no"
|
|
219
|
+
md.settings[k] = str(v) if not isinstance(v, list) else v
|
|
220
|
+
if "sef_meta" in site:
|
|
221
|
+
md.site["notes"] = site["sef_meta"]
|
|
222
|
+
for var, ov in j.get("variables", {}).items():
|
|
223
|
+
if var.startswith("_"):
|
|
224
|
+
continue
|
|
225
|
+
ov = {k: v for k, v in ov.items() if not k.startswith("_")}
|
|
226
|
+
if "unit" in ov:
|
|
227
|
+
md.units[var] = ov.pop("unit")
|
|
228
|
+
md.variables[var] = ov
|
|
229
|
+
for col, ov in j.get("columns", {}).items():
|
|
230
|
+
if not col.startswith("_"):
|
|
231
|
+
md.columns[" ".join(col.split())] = {k: v for k, v in ov.items() if not k.startswith("_")}
|
|
232
|
+
md.wind_words = {k.lower(): v for k, v in j.get("wind_words", {}).items() if not k.startswith("_")}
|
|
233
|
+
md.weather_terms = [(k.lower(), v, "exact") for k, v in j.get("weather_terms", {}).items()
|
|
234
|
+
if not k.startswith("_")]
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def write_template(path, variables=None):
|
|
238
|
+
"""Write a blank metadata csv. variables: list of (code, suggested unit, description)."""
|
|
239
|
+
with open(path, "w", encoding="utf-8", newline="") as f:
|
|
240
|
+
w = csv.writer(f)
|
|
241
|
+
w.writerow(["key", "value"])
|
|
242
|
+
for k, v in TEMPLATE:
|
|
243
|
+
w.writerow([k, v])
|
|
244
|
+
for code, unit, desc in (variables or []):
|
|
245
|
+
w.writerow([f"unit.{code}", unit, f"# {desc}"])
|