rda-python-icoads 1.0.9__py3-none-any.whl → 1.0.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rda-python-icoads might be problematic. Click here for more details.
- rda_python_icoads/PgIMMA.py +142 -102
- rda_python_icoads/countattm.py +125 -0
- rda_python_icoads/countattmvar.py +237 -0
- rda_python_icoads/countsst.py +221 -0
- rda_python_icoads/fillicoads.py +15 -8
- rda_python_icoads/fixiidx.py +70 -0
- rda_python_icoads/maxsst.py +262 -0
- rda_python_icoads/msg3_subset.py +1 -1
- {rda_python_icoads-1.0.9.dist-info → rda_python_icoads-1.0.11.dist-info}/METADATA +3 -2
- {rda_python_icoads-1.0.9.dist-info → rda_python_icoads-1.0.11.dist-info}/RECORD +14 -9
- {rda_python_icoads-1.0.9.dist-info → rda_python_icoads-1.0.11.dist-info}/WHEEL +1 -1
- {rda_python_icoads-1.0.9.dist-info → rda_python_icoads-1.0.11.dist-info}/entry_points.txt +5 -0
- {rda_python_icoads-1.0.9.dist-info → rda_python_icoads-1.0.11.dist-info/licenses}/LICENSE +0 -0
- {rda_python_icoads-1.0.9.dist-info → rda_python_icoads-1.0.11.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
##################################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title : countattm
|
|
6
|
+
# Author : Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date : 01/09/2021
|
|
8
|
+
# 2025-03-04 transferred to package rda_python_icoads from
|
|
9
|
+
# https://github.com/NCAR/rda-icoads.git
|
|
10
|
+
# Purpose : process ICOADS data files in IMMA format and count the matching,
|
|
11
|
+
# unmatching and empty records
|
|
12
|
+
#
|
|
13
|
+
# Github : https://github.com/NCAR/rda-python-icoads.git
|
|
14
|
+
#
|
|
15
|
+
##################################################################################
|
|
16
|
+
|
|
17
|
+
import sys
|
|
18
|
+
import re
|
|
19
|
+
from rda_python_common import PgLOG
|
|
20
|
+
from rda_python_common import PgDBI
|
|
21
|
+
from . import PgIMMA
|
|
22
|
+
|
|
23
|
+
PVALS = {
|
|
24
|
+
'group' : None,
|
|
25
|
+
'files' : [],
|
|
26
|
+
'aname' : None,
|
|
27
|
+
'bym' : None,
|
|
28
|
+
'eym' : None
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
ACOUNTS = {}
|
|
32
|
+
|
|
33
|
+
#
|
|
34
|
+
# main function
|
|
35
|
+
#
|
|
36
|
+
def main():
|
|
37
|
+
|
|
38
|
+
option = ''
|
|
39
|
+
argv = sys.argv[1:]
|
|
40
|
+
|
|
41
|
+
for arg in argv:
|
|
42
|
+
if arg == "-b":
|
|
43
|
+
PgLOG.PGLOG['BCKGRND'] = 1
|
|
44
|
+
elif arg == '-g':
|
|
45
|
+
option = 'g'
|
|
46
|
+
elif re.match(r'^-', arg):
|
|
47
|
+
PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
|
|
48
|
+
elif option:
|
|
49
|
+
PVALS['group'] = arg
|
|
50
|
+
option = ''
|
|
51
|
+
else:
|
|
52
|
+
PVALS['files'].append(arg)
|
|
53
|
+
|
|
54
|
+
if not (PVALS['files'] and re.match(r'^(monthly|yearly)$', PVALS['group'])):
|
|
55
|
+
print("Usage: countattm -g GroupBy (monthly|yearly) FileNameList")
|
|
56
|
+
print(" Group by Monthly or Yearly is mandatory")
|
|
57
|
+
print(" At least one file name needs to be present to count icoads attm data")
|
|
58
|
+
sys.exit(0)
|
|
59
|
+
|
|
60
|
+
PgLOG.PGLOG['LOGFILE'] = "icoads.log"
|
|
61
|
+
PgDBI.ivaddb_dbname()
|
|
62
|
+
PgLOG.cmdlog("countattm {}".format(' '.join(argv)))
|
|
63
|
+
for file in PVALS['files']: count_attm_file(file)
|
|
64
|
+
dump_attm_counts()
|
|
65
|
+
PgLOG.cmdlog()
|
|
66
|
+
sys.exit(0)
|
|
67
|
+
|
|
68
|
+
#
|
|
69
|
+
# read icoads record from given file name and count the records
|
|
70
|
+
#
|
|
71
|
+
def count_attm_file(fname):
|
|
72
|
+
|
|
73
|
+
PgLOG.pglog("Count attm records in File '{}'".format(fname), PgLOG.WARNLG)
|
|
74
|
+
|
|
75
|
+
# Get file month
|
|
76
|
+
ms = re.search(r'(\d\d\d\d)-(\d\d)', fname)
|
|
77
|
+
if ms:
|
|
78
|
+
yr = ms.group(1)
|
|
79
|
+
mn = ms.group(2)
|
|
80
|
+
ym = "{}-{}".format(yr, mn)
|
|
81
|
+
if not PVALS['bym']: PVALS['bym'] = ym
|
|
82
|
+
PVALS['eym'] = ym
|
|
83
|
+
key = yr if PVALS['group'] == "yearly" else ym
|
|
84
|
+
if key not in ACOUNTS:
|
|
85
|
+
ACOUNTS[key] = {'match' : 0, 'unmatch' : 0, 'empty' : 0, 'total' : 0}
|
|
86
|
+
else:
|
|
87
|
+
PgLOG.pglog(fname + ": miss year/month values in file name", PgLOG.LGEREX)
|
|
88
|
+
|
|
89
|
+
ATTM = open(fname, 'r')
|
|
90
|
+
acnt = 0
|
|
91
|
+
line = ATTM.readline()
|
|
92
|
+
# check and record standalone attm name
|
|
93
|
+
if not PVALS['aname']: PVALS['aname'] = PgIMMA.identify_attm_name(line)
|
|
94
|
+
while line:
|
|
95
|
+
ACOUNTS[key]['total'] += 1
|
|
96
|
+
# commet out these two line for normal records
|
|
97
|
+
line = line.rstrip()
|
|
98
|
+
if len(line) < 20:
|
|
99
|
+
ACOUNTS[key]['empty'] += 1
|
|
100
|
+
else:
|
|
101
|
+
idate = PgIMMA.get_imma_date(line)
|
|
102
|
+
if idate or idate is None:
|
|
103
|
+
ACOUNTS[key]['match'] += 1
|
|
104
|
+
else:
|
|
105
|
+
ACOUNTS[key]['unmatch'] += 1
|
|
106
|
+
line = ATTM.readline()
|
|
107
|
+
ATTM.close()
|
|
108
|
+
|
|
109
|
+
#
|
|
110
|
+
# dump attm counts by group
|
|
111
|
+
#
|
|
112
|
+
def dump_attm_counts():
|
|
113
|
+
|
|
114
|
+
fname = "{}_COUNTS_{}_{}-{}.txt".format(PVALS['aname'], PVALS['group'].upper(), PVALS['bym'], PVALS['eym'])
|
|
115
|
+
ATTM = open(fname, 'w')
|
|
116
|
+
ATTM.write(PVALS['group'] + ", match, unmatch, empty, total\n")
|
|
117
|
+
|
|
118
|
+
for key in sorted(ACOUNTS):
|
|
119
|
+
ATTM.write("{}, {}, {}, {}, {}\n".format(key, ACOUNTS[key]['match'],
|
|
120
|
+
ACOUNTS[key]['unmatch'], ACOUNTS[key]['empty'], ACOUNTS[key]['total']))
|
|
121
|
+
|
|
122
|
+
#
|
|
123
|
+
# call main() to start program
|
|
124
|
+
#
|
|
125
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
##################################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title : countattmvar
|
|
6
|
+
# Author : Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date : 01/09/2021
|
|
8
|
+
# 2025-03-04 transferred to package rda_python_icoads from
|
|
9
|
+
# https://github.com/NCAR/rda-icoads.git
|
|
10
|
+
# Purpose : read ICOADS data from IVADDB and count out daily, monthly or year records
|
|
11
|
+
# by attms/variable
|
|
12
|
+
#
|
|
13
|
+
# Github : https://github.com/NCAR/rda-python-icoads.git
|
|
14
|
+
#
|
|
15
|
+
##################################################################################
|
|
16
|
+
|
|
17
|
+
import sys
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
from os import path as op
|
|
21
|
+
from rda_python_common import PgLOG
|
|
22
|
+
from rda_python_common import PgDBI
|
|
23
|
+
from rda_python_common import PgUtil
|
|
24
|
+
from . import PgIMMA
|
|
25
|
+
|
|
26
|
+
PVALS = {
|
|
27
|
+
'bdate' : None,
|
|
28
|
+
'edate' : None,
|
|
29
|
+
'bpdate' : [],
|
|
30
|
+
'epdate' : [],
|
|
31
|
+
'period' : [],
|
|
32
|
+
'group' : None,
|
|
33
|
+
'aname' : None,
|
|
34
|
+
'jname' : None,
|
|
35
|
+
'fname' : None,
|
|
36
|
+
'oname' : None,
|
|
37
|
+
'wcnd' : None,
|
|
38
|
+
'atables' : {}, # cache atable names processed. value 1 table exists; 0 not exsits
|
|
39
|
+
'jtables' : {} # cache jtable names processed. value 1 table exists; 0 not exsits
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
#
|
|
43
|
+
# main function to run dsarch
|
|
44
|
+
#
|
|
45
|
+
def main():
|
|
46
|
+
|
|
47
|
+
option = None
|
|
48
|
+
argv = sys.argv[1:]
|
|
49
|
+
|
|
50
|
+
for arg in argv:
|
|
51
|
+
if arg == "-b":
|
|
52
|
+
PgLOG.PGLOG['BCKGRND'] = 1
|
|
53
|
+
continue
|
|
54
|
+
ms = re.match(r'^-[afgjow])$', arg)
|
|
55
|
+
if ms:
|
|
56
|
+
option = ms.group(1)
|
|
57
|
+
continue
|
|
58
|
+
if re.match(r'^-', arg): PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
|
|
59
|
+
elif option:
|
|
60
|
+
if option == 'a':
|
|
61
|
+
PVALS['aname'] = arg
|
|
62
|
+
elif option == 'g':
|
|
63
|
+
PVALS['group'] = arg
|
|
64
|
+
elif option == 'j':
|
|
65
|
+
PVALS['jname'] = arg
|
|
66
|
+
elif option == 'f':
|
|
67
|
+
PVALS['fname'] = arg
|
|
68
|
+
elif option == 'o':
|
|
69
|
+
PVALS['oname'] = arg
|
|
70
|
+
elif option == 'w':
|
|
71
|
+
PVALS['wcnd'] = arg
|
|
72
|
+
option = ''
|
|
73
|
+
elif not PVALS['bdate']:
|
|
74
|
+
PVALS['bdate'] = arg
|
|
75
|
+
elif not PVALS['edate']:
|
|
76
|
+
PVALS['edate'] = arg
|
|
77
|
+
else:
|
|
78
|
+
PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
|
|
79
|
+
|
|
80
|
+
PgDBI.ivaddb_dbname()
|
|
81
|
+
|
|
82
|
+
if(not (PVALS['bdate'] and PVALS['edate']) or not re.match(r'^(daily|monthly|yearly)$', PVALS['group']) or
|
|
83
|
+
not (PVALS['aname'] and PVALS['fname'] and PVALS['wcnd'])):
|
|
84
|
+
pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
|
|
85
|
+
print("Usage: countattmvar -g GroupBy (daily|monthly|yearly) -a AttmName -f FieldName -w WhereCondition [-j JoinName] BeginDate EndDate")
|
|
86
|
+
print(" Group by Daily, Monthly or Yearly is mandatory")
|
|
87
|
+
print(" Set BeginDate and EndDate between '' and '{}'".format(pgrec['bdate'], pgrec['edate']))
|
|
88
|
+
sys.exit(0)
|
|
89
|
+
|
|
90
|
+
if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
|
|
91
|
+
tmpdate = PVALS['bdate']
|
|
92
|
+
PVALS['bdate'] = PVALS['edate']
|
|
93
|
+
PVALS['edate'] = tmpdate
|
|
94
|
+
|
|
95
|
+
if PVALS['jname'] and (PVALS['jname'] == PVALS['aname'] or PVALS['jname'] == "icoreloc"): PVALS['jname'] = None
|
|
96
|
+
|
|
97
|
+
PgLOG.PGLOG['LOGFILE'] = "icoads.log"
|
|
98
|
+
PgLOG.cmdlog("countattmvar {}".format(' '.join(argv)))
|
|
99
|
+
|
|
100
|
+
if not PVALS['oname']:
|
|
101
|
+
PVALS['oname'] = "{}.{}_COUNTS_{}_{}-{}.txt".format(PVALS['aname'], PVALS['fname'],
|
|
102
|
+
PVALS['group'].upper(), PVALS['bdate'], PVALS['edate'])
|
|
103
|
+
|
|
104
|
+
IMMA = open(PVALS['oname'], 'w')
|
|
105
|
+
IMMA.write("{}, {}\n".format(PVALS['group'], PVALS['fname']))
|
|
106
|
+
count_attm_variable(IMMA)
|
|
107
|
+
IMMA.close()
|
|
108
|
+
PgLOG.PgLOG.cmdlog()
|
|
109
|
+
sys.exit(0)
|
|
110
|
+
|
|
111
|
+
#
|
|
112
|
+
# count the variable of a attm daily/month/y/yearly
|
|
113
|
+
#
|
|
114
|
+
def count_attm_variable(IMMA):
|
|
115
|
+
|
|
116
|
+
pcnt = init_periods()
|
|
117
|
+
tcnt = 0
|
|
118
|
+
for pidx in range(pcnt):
|
|
119
|
+
if PVALS['group'] == 'daily':
|
|
120
|
+
acnt = count_daily_attm_variable(PVALS['bpdate'][pidx])
|
|
121
|
+
else:
|
|
122
|
+
acnt = count_period_attm_variable(pidx)
|
|
123
|
+
if not acnt: continue
|
|
124
|
+
IMMA.write("{}, {}\n".format(PVALS['period'][pidx], acnt))
|
|
125
|
+
tcnt += acnt
|
|
126
|
+
|
|
127
|
+
if pcnt > 1:
|
|
128
|
+
IMMA.write("Total, {}\n".format(tcnt))
|
|
129
|
+
PgLOG.pglog("{}.{}: {} total for {} {} periods".format(PVALS['aname'], PVALS['fname'], tcnt, pcnt, PVALS['group']), PgLOG.LOGWRN)
|
|
130
|
+
|
|
131
|
+
#
|
|
132
|
+
# read icoads record from given file name and save them into RDADB
|
|
133
|
+
#
|
|
134
|
+
def count_period_attm_variable(pidx):
|
|
135
|
+
|
|
136
|
+
bdate = PVALS['bpdate'][pidx]
|
|
137
|
+
edate = PVALS['epdate'][pidx]
|
|
138
|
+
btidx = PgIMMA.date2tidx(bdate)
|
|
139
|
+
etidx = PgIMMA.date2tidx(edate)
|
|
140
|
+
tblcnt = etidx-btidx+1
|
|
141
|
+
PgLOG.pglog("Counting {}.{} for {} from IVADDB".format(PVALS['aname'], PVALS['fname'], PVALS['period'][pidx]), PgLOG.WARNLG)
|
|
142
|
+
|
|
143
|
+
# get acount from the first table
|
|
144
|
+
acount = 0
|
|
145
|
+
if tblcnt == 1:
|
|
146
|
+
cnds = ["date BETWEEN '{}' AND '{}' AND ".format(bdate, edate)]
|
|
147
|
+
else:
|
|
148
|
+
cnds = ['']*tblcnt
|
|
149
|
+
cnds[0] = "date >='{}' AND ".format(bdate)
|
|
150
|
+
cnds[tblcnt-1] = "date <= '{}' AND ".format(edate)
|
|
151
|
+
for i in range(tblcnt): acount += count_table_attm_variable(cnds[i], i + btidx)
|
|
152
|
+
|
|
153
|
+
if acount > 0:
|
|
154
|
+
PgLOG.pglog("{}.{}: {} for {} of {}".format(PVALS['aname'], PVALS['fname'], acount, PVALS['wcnd'], PVALS['period'][pidx]), PgLOG.LOGWRN)
|
|
155
|
+
return acount
|
|
156
|
+
|
|
157
|
+
#
|
|
158
|
+
# count IMMA records for given date
|
|
159
|
+
#
|
|
160
|
+
def count_daily_attm_variable(cdate):
|
|
161
|
+
|
|
162
|
+
tidx = PgIMMA.date2tidx(cdate)
|
|
163
|
+
if not tidx: return 0
|
|
164
|
+
PgLOG.pglog("Counting {}.{} for {} from IVADDB".format(PVALS['aname'], PVALS['fname'], cdate), PgLOG.WARNLG)
|
|
165
|
+
acount = count_table_attm_variable("date = '{}' AND ".format(cdate), tidx)
|
|
166
|
+
if acount > 0:
|
|
167
|
+
PgLOG.pglog("{}.{}: {} for {} of ".format(PVALS['aname'], PVALS['fname'], acount, PVALS['wcnd'], cdate), PgLOG.LOGWRN)
|
|
168
|
+
return acount
|
|
169
|
+
|
|
170
|
+
#
|
|
171
|
+
# count IMMA records from one table
|
|
172
|
+
#
|
|
173
|
+
def count_table_attm_variable(dcnd, tidx):
|
|
174
|
+
|
|
175
|
+
if PVALS['jname']:
|
|
176
|
+
jtable = "{}_{}".format(PVALS['jname'], tidx)
|
|
177
|
+
if tidx not in PVALS['jtables']: PVALS['jtables'][tidx] = PgDBI.pgcheck(jtable)
|
|
178
|
+
if not PVALS['jtables'][tidx]: return 0
|
|
179
|
+
|
|
180
|
+
atable = "{}_{}".format(PVALS['aname'], tidx)
|
|
181
|
+
if tidx not in PVALS['atables']: PVALS['atables'][tidx] = PgDBI.pgcheck(atable)
|
|
182
|
+
if not PVALS['atables'][tidx]: return 0
|
|
183
|
+
|
|
184
|
+
if PVALS['aname'] == 'icoreloc':
|
|
185
|
+
if PVALS['jname']:
|
|
186
|
+
table = "{} j, {} n".format(jtable, atable)
|
|
187
|
+
jcnd = "j.iidx = n.iidx AND "
|
|
188
|
+
else:
|
|
189
|
+
table = atable
|
|
190
|
+
jcnd = ""
|
|
191
|
+
else:
|
|
192
|
+
mtable = "icoreloc_{}".format(tidx)
|
|
193
|
+
if PVALS['jname']:
|
|
194
|
+
table = "{}m, {} j. {} n".format(mtable, jtable, atable)
|
|
195
|
+
jcnd = "m.iidx = j.iidx AND m.iidx = n.iidx AND "
|
|
196
|
+
else:
|
|
197
|
+
table = "{} m, {} n".format(mtable, atable)
|
|
198
|
+
cnd = "{} AND m.iidx = n.iidx AND "
|
|
199
|
+
|
|
200
|
+
return PgDBI.pgget(table, "", dcnd + jcnd + PVALS['wcnd'], PgLOG.LGEREX)
|
|
201
|
+
|
|
202
|
+
#
|
|
203
|
+
# initialize the group periods
|
|
204
|
+
#
|
|
205
|
+
def init_periods():
|
|
206
|
+
|
|
207
|
+
bdate = PVALS['bdate']
|
|
208
|
+
|
|
209
|
+
if PVALS['group'] == "yearly":
|
|
210
|
+
dfmt = "YYYY"
|
|
211
|
+
eflg = "Y"
|
|
212
|
+
elif PVALS['group'] == 'monthly':
|
|
213
|
+
dfmt = "YYYY-MM"
|
|
214
|
+
eflg = "M"
|
|
215
|
+
else: # must be daily
|
|
216
|
+
dfmt = "YYYY-MM-DD"
|
|
217
|
+
eflg = ""
|
|
218
|
+
|
|
219
|
+
pcnt = 0
|
|
220
|
+
while True:
|
|
221
|
+
pcnt += 1
|
|
222
|
+
PVALS['bpdate'].append(bdate)
|
|
223
|
+
PVALS['period'].append(PgUtil.format_date(bdate, dfmt))
|
|
224
|
+
edate = PgUtil.enddate(bdate, 0, eflg) if eflg else bdate
|
|
225
|
+
if PgUtil.diffdate(edate, PVALS['edate']) < 0:
|
|
226
|
+
PVALS['epdate'].append(edate)
|
|
227
|
+
bdate = PgUtil.adddate(edate, 0, 0, 1)
|
|
228
|
+
else:
|
|
229
|
+
PVALS['epdate'].append(PVALS['edate'])
|
|
230
|
+
break
|
|
231
|
+
|
|
232
|
+
return pcnt
|
|
233
|
+
|
|
234
|
+
#
|
|
235
|
+
# call main() to start program
|
|
236
|
+
#
|
|
237
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
##################################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title : countsst
|
|
6
|
+
# Author : Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date : 01/09/2021
|
|
8
|
+
# 2025-03-04 transferred to package rda_python_icoads from
|
|
9
|
+
# https://github.com/NCAR/rda-icoads.git
|
|
10
|
+
# Purpose : read ICOADS data from IVADDB and count SST records by dail, monthly
|
|
11
|
+
# or yearly
|
|
12
|
+
#
|
|
13
|
+
# Github : https://github.com/NCAR/rda-python-icoads.git
|
|
14
|
+
#
|
|
15
|
+
##################################################################################
|
|
16
|
+
|
|
17
|
+
import sys
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
from os import path as op
|
|
21
|
+
from rda_python_common import PgLOG
|
|
22
|
+
from rda_python_common import PgDBI
|
|
23
|
+
from rda_python_common import PgUtil
|
|
24
|
+
from . import PgIMMA
|
|
25
|
+
|
|
26
|
+
PVALS = {
|
|
27
|
+
'bdate' : None,
|
|
28
|
+
'edate' : None,
|
|
29
|
+
'bpdate' : [],
|
|
30
|
+
'epdate' : [],
|
|
31
|
+
'period' : [],
|
|
32
|
+
'group' : None,
|
|
33
|
+
'oname' : None
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
#
|
|
37
|
+
# main function to run dsarch
|
|
38
|
+
#
|
|
39
|
+
def main():
|
|
40
|
+
|
|
41
|
+
option = None
|
|
42
|
+
argv = sys.argv[1:]
|
|
43
|
+
|
|
44
|
+
for arg in argv:
|
|
45
|
+
if arg == "-b":
|
|
46
|
+
PgLOG.PGLOG['BCKGRND'] = 1
|
|
47
|
+
continue
|
|
48
|
+
ms = re.match(r'-([go])$', arg)
|
|
49
|
+
if ms:
|
|
50
|
+
option = ms.group(1)
|
|
51
|
+
continue
|
|
52
|
+
if re.match(r'^-', arg): PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
|
|
53
|
+
elif option:
|
|
54
|
+
if option == 'g':
|
|
55
|
+
PVALS['group'] = arg
|
|
56
|
+
elif option == 'o':
|
|
57
|
+
PVALS['oname'] = arg
|
|
58
|
+
option = ''
|
|
59
|
+
elif not PVALS['bdate']:
|
|
60
|
+
PVALS['bdate'] = arg
|
|
61
|
+
elif not PVALS['edate']:
|
|
62
|
+
PVALS['edate'] = arg
|
|
63
|
+
else:
|
|
64
|
+
PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
|
|
65
|
+
|
|
66
|
+
PgDBI.ivaddb_dbname()
|
|
67
|
+
|
|
68
|
+
if not (PVALS['bdate'] and PVALS['edate'] and re.match(r'^(daily|monthly|yearly)$', PVALS['group'])):
|
|
69
|
+
pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
|
|
70
|
+
print("Usage: countsst -g GroupBy (daily|monthly|yearly) BeginDate EndDate")
|
|
71
|
+
print(" Group by Daily, Monthly or Yearly is mandatory")
|
|
72
|
+
print(" Set BeginDate and EndDate between '{}' and '{}'".format(pgrec['bdate'], pgrec['edate']))
|
|
73
|
+
sys.exit(0)
|
|
74
|
+
|
|
75
|
+
if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
|
|
76
|
+
tmpdate = PVALS['bdate']
|
|
77
|
+
PVALS['bdate'] = PVALS['edate']
|
|
78
|
+
PVALS['edate'] = tmpdate
|
|
79
|
+
|
|
80
|
+
PgLOG.PGLOG['LOGFILE'] = "icoads.log"
|
|
81
|
+
PgLOG.cmdlog("countsst {}".format(' '.join(argv)))
|
|
82
|
+
|
|
83
|
+
if not PVALS['oname']:
|
|
84
|
+
PVALS['oname'] = "SST_COUNTS_{}_{}-{}.txt".format(PVALS['group'].upper(), PVALS['bdate'], PVALS['edate'])
|
|
85
|
+
|
|
86
|
+
IMMA = open(PVALS['oname'], 'w')
|
|
87
|
+
IMMA.write(PVALS['group'] + ", NOCN, ENH, SST, TOTAL\n")
|
|
88
|
+
count_sst(IMMA)
|
|
89
|
+
IMMA.close()
|
|
90
|
+
PgLOG.cmdlog()
|
|
91
|
+
sys.exit(0)
|
|
92
|
+
|
|
93
|
+
#
|
|
94
|
+
# count the SST values daily/monthly/yearly
|
|
95
|
+
#
|
|
96
|
+
def count_sst(fd):
|
|
97
|
+
|
|
98
|
+
pcnt = init_periods()
|
|
99
|
+
tcnts = [0]*4
|
|
100
|
+
getdaily = 1 if PVALS['group'] == 'daily' else 0
|
|
101
|
+
|
|
102
|
+
for pidx in range(pcnt):
|
|
103
|
+
if getdaily:
|
|
104
|
+
acnts = count_daily_sst(PVALS['bpdate'][pidx])
|
|
105
|
+
else:
|
|
106
|
+
acnts = count_period_sst(pidx)
|
|
107
|
+
|
|
108
|
+
line = PVALS['period'][pidx]
|
|
109
|
+
for i in range(4):
|
|
110
|
+
line += ", {}".format(acnts[i])
|
|
111
|
+
tcnts[i] += acnts[i]
|
|
112
|
+
fd.write(line + "\n")
|
|
113
|
+
|
|
114
|
+
if pcnt > 1:
|
|
115
|
+
line = "Total"
|
|
116
|
+
for i in range(4):
|
|
117
|
+
line += ", {}".format(tcnts[i])
|
|
118
|
+
fd.write(line + "\n")
|
|
119
|
+
PgLOG.pglog("{} total for {} {} periods".format(tcnts[3], pcnt, PVALS['group']), PgLOG.LOGWRN)
|
|
120
|
+
|
|
121
|
+
#
|
|
122
|
+
# read icoads record from given file name and save them into RDADB
|
|
123
|
+
#
|
|
124
|
+
def count_period_sst(pidx):
|
|
125
|
+
|
|
126
|
+
bdate = PVALS['bpdate'][pidx]
|
|
127
|
+
edate = PVALS['epdate'][pidx]
|
|
128
|
+
btidx = PgIMMA.date2tidx(bdate)
|
|
129
|
+
etidx = PgIMMA.date2tidx(edate)
|
|
130
|
+
tblcnt = etidx-btidx+1
|
|
131
|
+
acounts = [0]*4
|
|
132
|
+
acnds = ['']*tblcnt
|
|
133
|
+
mcnds = ['']*tblcnt
|
|
134
|
+
|
|
135
|
+
PgLOG.pglog("Counting SST for {} from IVADDB".format(PVALS['period'][pidx]), PgLOG.WARNLG)
|
|
136
|
+
|
|
137
|
+
if tblcnt == 1:
|
|
138
|
+
acnds[0] = "date BETWEEN '{}' AND '{}'".format(bdate, edate)
|
|
139
|
+
mcnds[0] = "time BETWEEN '{} 00:00:00' AND '{} 23:59:59'".format(bdate, edate)
|
|
140
|
+
else:
|
|
141
|
+
acnds[0] = "date >= '{}'".format(bdate)
|
|
142
|
+
mcnds[0] = "time >= '{} 00:00:00'".format(bdate)
|
|
143
|
+
acnds[tblcnt-1] = "date <= '{}'".format(edate)
|
|
144
|
+
mcnds[tblcnt-1] = "time <= '{} 23:59:59'".format(edate)
|
|
145
|
+
|
|
146
|
+
for i in range(tblcnt): count_table_sst(btidx+i, acounts, acnds[i], mcnds[i])
|
|
147
|
+
if acounts[3]: PgLOG.pglog("{} for {}".format(acounts[3], PVALS['period'][pidx]), PgLOG.LOGWRN)
|
|
148
|
+
|
|
149
|
+
return acounts
|
|
150
|
+
|
|
151
|
+
#
|
|
152
|
+
# count IMMA records for given date
|
|
153
|
+
#
|
|
154
|
+
def count_daily_sst(cdate):
|
|
155
|
+
|
|
156
|
+
tidx = PgIMMA.date2tidx(cdate)
|
|
157
|
+
acounts = [0]*4
|
|
158
|
+
|
|
159
|
+
PgLOG.pglog("Counting SST for {} from IVADDB".format(cdate), PgLOG.WARNLG)
|
|
160
|
+
acnd = "date = '{}'".format(cdate)
|
|
161
|
+
mcnd = "time BETWEEN '{} 00:00:00' AND '{} 23:59:59'".format(cdate, cdate)
|
|
162
|
+
count_table_sst(tidx, acounts, acnd, mcnd)
|
|
163
|
+
if acounts[3]: PgLOG.pglog("{} for {}".format(acounts[3], cdate), PgLOG.LOGWRN)
|
|
164
|
+
|
|
165
|
+
return acounts
|
|
166
|
+
|
|
167
|
+
#
|
|
168
|
+
# count IMMA records from one table
|
|
169
|
+
#
|
|
170
|
+
def count_table_sst(tidx, acounts, acnd, mcnd):
|
|
171
|
+
|
|
172
|
+
table = "icoreloc_{}".format(tidx)
|
|
173
|
+
cnt = PgDBI.pgget(table, "", acnd, PgLOG.LGEREX)
|
|
174
|
+
if not cnt: return acounts
|
|
175
|
+
acounts[3] += cnt
|
|
176
|
+
|
|
177
|
+
table = 'domsdb.idoms_{}'.format(tidx)
|
|
178
|
+
if mcnd: mcnd += ' AND '
|
|
179
|
+
mcnd += "sea_water_temperature IS NOT NULL"
|
|
180
|
+
acounts[2] += PgDBI.pgget(table, "", mcnd, PgLOG.LGEREX)
|
|
181
|
+
acnd = mcnd + " AND sea_water_temperature_quality = 0"
|
|
182
|
+
acounts[1] += PgDBI.pgget(table, "", acnd, PgLOG.LGEREX)
|
|
183
|
+
acnd = mcnd + " AND sea_water_temperature_depth > 0"
|
|
184
|
+
acounts[0] += PgDBI.pgget(table, "", acnd, PgLOG.LGEREX)
|
|
185
|
+
|
|
186
|
+
return acounts
|
|
187
|
+
|
|
188
|
+
#
|
|
189
|
+
# initialize period list
|
|
190
|
+
#
|
|
191
|
+
def init_periods():
|
|
192
|
+
|
|
193
|
+
bdate = PVALS['bdate']
|
|
194
|
+
if PVALS['group'] == "yearly":
|
|
195
|
+
dfmt = "YYYY"
|
|
196
|
+
eflg = "Y"
|
|
197
|
+
elif PVALS['group'] == 'monthly':
|
|
198
|
+
dfmt = "YYYY-MM"
|
|
199
|
+
eflg = "M"
|
|
200
|
+
else: # must be daily
|
|
201
|
+
dfmt = "YYYY-MM-DD"
|
|
202
|
+
eflg = ""
|
|
203
|
+
pcnt = 0
|
|
204
|
+
while True:
|
|
205
|
+
pcnt += 1
|
|
206
|
+
PVALS['bpdate'].append(bdate)
|
|
207
|
+
PVALS['period'].append(PgUtil.format_date(bdate, dfmt))
|
|
208
|
+
edate = PgUtil.enddate(bdate, 0, eflg) if eflg else bdate
|
|
209
|
+
if PgUtil.diffdate(PVALS['edate'], edate) > 0:
|
|
210
|
+
PVALS['epdate'].append(edate)
|
|
211
|
+
bdate = PgUtil.adddate(edate, 0, 0, 1)
|
|
212
|
+
else:
|
|
213
|
+
PVALS['epdate'].append(PVALS['edate'])
|
|
214
|
+
break
|
|
215
|
+
|
|
216
|
+
return pcnt
|
|
217
|
+
|
|
218
|
+
#
|
|
219
|
+
# call main() to start program
|
|
220
|
+
#
|
|
221
|
+
if __name__ == "__main__": main()
|
rda_python_icoads/fillicoads.py
CHANGED
|
@@ -33,8 +33,10 @@ PVALS = {
|
|
|
33
33
|
def main():
|
|
34
34
|
|
|
35
35
|
addinventory = leaduid = chkexist = 0
|
|
36
|
+
rn3 = -1
|
|
36
37
|
argv = sys.argv[1:]
|
|
37
|
-
|
|
38
|
+
|
|
39
|
+
option = None
|
|
38
40
|
for arg in argv:
|
|
39
41
|
if arg == "-b":
|
|
40
42
|
PgLOG.PGLOG['BCKGRND'] = 1
|
|
@@ -44,27 +46,32 @@ def main():
|
|
|
44
46
|
leaduid = 1
|
|
45
47
|
elif arg == "-e":
|
|
46
48
|
chkexist = 1
|
|
49
|
+
elif arg == "-r":
|
|
50
|
+
option = 'r'
|
|
47
51
|
elif arg == "-i":
|
|
48
52
|
addinventory = 1
|
|
49
53
|
elif re.match(r'^-', arg):
|
|
50
54
|
PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
|
|
55
|
+
elif option == 'r':
|
|
56
|
+
rn3 = int(arg)
|
|
51
57
|
else:
|
|
52
58
|
PVALS['files'].append(arg)
|
|
53
59
|
|
|
54
60
|
if not PVALS['files']:
|
|
55
|
-
print("Usage: fillicoads [-a] [-e] [-i] [-u] FileNameList")
|
|
61
|
+
print("Usage: fillicoads [-a] [-e] [-i] [-r] [-u] FileNameList")
|
|
56
62
|
print(" At least one file name needs to fill icoads data into Postgres Server")
|
|
57
63
|
print(" Option -a: add all attms, including multi-line ones, such as IVAD and REANQC")
|
|
58
64
|
print(" Option -i: add daily counting records into inventory table")
|
|
65
|
+
print(" Option -r: last digit of IMMA release number")
|
|
59
66
|
print(" Option -u: standalone attachment records with leading 6-character UID")
|
|
60
67
|
print(" Option -e: check existing record before adding attm")
|
|
61
68
|
sys.exit(0)
|
|
62
69
|
|
|
63
70
|
PgLOG.PGLOG['LOGFILE'] = "icoads.log"
|
|
64
|
-
PgDBI.
|
|
65
|
-
|
|
71
|
+
PgDBI.set_scname(dbname = 'ivaddb', scname = 'ivaddb1', lnname = 'ivaddb', dbhost = PgLOG.PGLOG['PMISCHOST'])
|
|
72
|
+
|
|
66
73
|
PgLOG.cmdlog("fillicoads {}".format(' '.join(argv)))
|
|
67
|
-
PgIMMA.init_current_indices(leaduid, chkexist)
|
|
74
|
+
PgIMMA.init_current_indices(leaduid, chkexist, rn3)
|
|
68
75
|
PVALS['names'] = '/'.join(PgIMMA.IMMA_NAMES)
|
|
69
76
|
fill_imma_data(addinventory)
|
|
70
77
|
PgLOG.cmdlog()
|
|
@@ -103,14 +110,14 @@ def process_imma_file(fname, addinventory):
|
|
|
103
110
|
idate = cdate = PgIMMA.get_imma_date(line)
|
|
104
111
|
if cdate:
|
|
105
112
|
PgIMMA.init_indices_for_date(cdate, iname)
|
|
106
|
-
records = PgIMMA.get_imma_records(
|
|
113
|
+
records = PgIMMA.get_imma_records(cdate, line, records)
|
|
107
114
|
break
|
|
108
115
|
line = IMMA.readline()
|
|
109
116
|
|
|
110
117
|
line = IMMA.readline()
|
|
111
118
|
while line:
|
|
112
119
|
if PVALS['uatti'] and line[0:2] == PVALS['uatti']:
|
|
113
|
-
records = PgIMMA.get_imma_multiple_records(line, records)
|
|
120
|
+
records = PgIMMA.get_imma_multiple_records(cdate, line, records)
|
|
114
121
|
else:
|
|
115
122
|
idate = PgIMMA.get_imma_date(line)
|
|
116
123
|
if idate:
|
|
@@ -120,7 +127,7 @@ def process_imma_file(fname, addinventory):
|
|
|
120
127
|
records = {}
|
|
121
128
|
cdate = idate
|
|
122
129
|
PgIMMA.init_indices_for_date(cdate, iname)
|
|
123
|
-
records = PgIMMA.get_imma_records(
|
|
130
|
+
records = PgIMMA.get_imma_records(idate, line, records)
|
|
124
131
|
line = IMMA.readline()
|
|
125
132
|
|
|
126
133
|
IMMA.close()
|