rda-python-icoads 1.0.9__py3-none-any.whl → 1.0.11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rda-python-icoads might be problematic. Click here for more details.

@@ -0,0 +1,125 @@
1
+ #!/usr/bin/env python3
2
+ #
3
+ ##################################################################################
4
+ #
5
+ # Title : countattm
6
+ # Author : Zaihua Ji, zji@ucar.edu
7
+ # Date : 01/09/2021
8
+ # 2025-03-04 transferred to package rda_python_icoads from
9
+ # https://github.com/NCAR/rda-icoads.git
10
+ # Purpose : process ICOADS data files in IMMA format and count the matching,
11
+ # unmatching and empty records
12
+ #
13
+ # Github : https://github.com/NCAR/rda-python-icoads.git
14
+ #
15
+ ##################################################################################
16
+
17
+ import sys
18
+ import re
19
+ from rda_python_common import PgLOG
20
+ from rda_python_common import PgDBI
21
+ from . import PgIMMA
22
+
23
+ PVALS = {
24
+ 'group' : None,
25
+ 'files' : [],
26
+ 'aname' : None,
27
+ 'bym' : None,
28
+ 'eym' : None
29
+ }
30
+
31
+ ACOUNTS = {}
32
+
33
+ #
34
+ # main function
35
+ #
36
+ def main():
37
+
38
+ option = ''
39
+ argv = sys.argv[1:]
40
+
41
+ for arg in argv:
42
+ if arg == "-b":
43
+ PgLOG.PGLOG['BCKGRND'] = 1
44
+ elif arg == '-g':
45
+ option = 'g'
46
+ elif re.match(r'^-', arg):
47
+ PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
48
+ elif option:
49
+ PVALS['group'] = arg
50
+ option = ''
51
+ else:
52
+ PVALS['files'].append(arg)
53
+
54
+ if not (PVALS['files'] and re.match(r'^(monthly|yearly)$', PVALS['group'])):
55
+ print("Usage: countattm -g GroupBy (monthly|yearly) FileNameList")
56
+ print(" Group by Monthly or Yearly is mandatory")
57
+ print(" At least one file name needs to be present to count icoads attm data")
58
+ sys.exit(0)
59
+
60
+ PgLOG.PGLOG['LOGFILE'] = "icoads.log"
61
+ PgDBI.ivaddb_dbname()
62
+ PgLOG.cmdlog("countattm {}".format(' '.join(argv)))
63
+ for file in PVALS['files']: count_attm_file(file)
64
+ dump_attm_counts()
65
+ PgLOG.cmdlog()
66
+ sys.exit(0)
67
+
68
+ #
69
+ # read icoads record from given file name and count the records
70
+ #
71
+ def count_attm_file(fname):
72
+
73
+ PgLOG.pglog("Count attm records in File '{}'".format(fname), PgLOG.WARNLG)
74
+
75
+ # Get file month
76
+ ms = re.search(r'(\d\d\d\d)-(\d\d)', fname)
77
+ if ms:
78
+ yr = ms.group(1)
79
+ mn = ms.group(2)
80
+ ym = "{}-{}".format(yr, mn)
81
+ if not PVALS['bym']: PVALS['bym'] = ym
82
+ PVALS['eym'] = ym
83
+ key = yr if PVALS['group'] == "yearly" else ym
84
+ if key not in ACOUNTS:
85
+ ACOUNTS[key] = {'match' : 0, 'unmatch' : 0, 'empty' : 0, 'total' : 0}
86
+ else:
87
+ PgLOG.pglog(fname + ": miss year/month values in file name", PgLOG.LGEREX)
88
+
89
+ ATTM = open(fname, 'r')
90
+ acnt = 0
91
+ line = ATTM.readline()
92
+ # check and record standalone attm name
93
+ if not PVALS['aname']: PVALS['aname'] = PgIMMA.identify_attm_name(line)
94
+ while line:
95
+ ACOUNTS[key]['total'] += 1
96
+ # commet out these two line for normal records
97
+ line = line.rstrip()
98
+ if len(line) < 20:
99
+ ACOUNTS[key]['empty'] += 1
100
+ else:
101
+ idate = PgIMMA.get_imma_date(line)
102
+ if idate or idate is None:
103
+ ACOUNTS[key]['match'] += 1
104
+ else:
105
+ ACOUNTS[key]['unmatch'] += 1
106
+ line = ATTM.readline()
107
+ ATTM.close()
108
+
109
+ #
110
+ # dump attm counts by group
111
+ #
112
+ def dump_attm_counts():
113
+
114
+ fname = "{}_COUNTS_{}_{}-{}.txt".format(PVALS['aname'], PVALS['group'].upper(), PVALS['bym'], PVALS['eym'])
115
+ ATTM = open(fname, 'w')
116
+ ATTM.write(PVALS['group'] + ", match, unmatch, empty, total\n")
117
+
118
+ for key in sorted(ACOUNTS):
119
+ ATTM.write("{}, {}, {}, {}, {}\n".format(key, ACOUNTS[key]['match'],
120
+ ACOUNTS[key]['unmatch'], ACOUNTS[key]['empty'], ACOUNTS[key]['total']))
121
+
122
+ #
123
+ # call main() to start program
124
+ #
125
+ if __name__ == "__main__": main()
@@ -0,0 +1,237 @@
1
+ #!/usr/bin/env python3
2
+ #
3
+ ##################################################################################
4
+ #
5
+ # Title : countattmvar
6
+ # Author : Zaihua Ji, zji@ucar.edu
7
+ # Date : 01/09/2021
8
+ # 2025-03-04 transferred to package rda_python_icoads from
9
+ # https://github.com/NCAR/rda-icoads.git
10
+ # Purpose : read ICOADS data from IVADDB and count out daily, monthly or year records
11
+ # by attms/variable
12
+ #
13
+ # Github : https://github.com/NCAR/rda-python-icoads.git
14
+ #
15
+ ##################################################################################
16
+
17
+ import sys
18
+ import os
19
+ import re
20
+ from os import path as op
21
+ from rda_python_common import PgLOG
22
+ from rda_python_common import PgDBI
23
+ from rda_python_common import PgUtil
24
+ from . import PgIMMA
25
+
26
+ PVALS = {
27
+ 'bdate' : None,
28
+ 'edate' : None,
29
+ 'bpdate' : [],
30
+ 'epdate' : [],
31
+ 'period' : [],
32
+ 'group' : None,
33
+ 'aname' : None,
34
+ 'jname' : None,
35
+ 'fname' : None,
36
+ 'oname' : None,
37
+ 'wcnd' : None,
38
+ 'atables' : {}, # cache atable names processed. value 1 table exists; 0 not exsits
39
+ 'jtables' : {} # cache jtable names processed. value 1 table exists; 0 not exsits
40
+ }
41
+
42
+ #
43
+ # main function to run dsarch
44
+ #
45
+ def main():
46
+
47
+ option = None
48
+ argv = sys.argv[1:]
49
+
50
+ for arg in argv:
51
+ if arg == "-b":
52
+ PgLOG.PGLOG['BCKGRND'] = 1
53
+ continue
54
+ ms = re.match(r'^-[afgjow])$', arg)
55
+ if ms:
56
+ option = ms.group(1)
57
+ continue
58
+ if re.match(r'^-', arg): PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
59
+ elif option:
60
+ if option == 'a':
61
+ PVALS['aname'] = arg
62
+ elif option == 'g':
63
+ PVALS['group'] = arg
64
+ elif option == 'j':
65
+ PVALS['jname'] = arg
66
+ elif option == 'f':
67
+ PVALS['fname'] = arg
68
+ elif option == 'o':
69
+ PVALS['oname'] = arg
70
+ elif option == 'w':
71
+ PVALS['wcnd'] = arg
72
+ option = ''
73
+ elif not PVALS['bdate']:
74
+ PVALS['bdate'] = arg
75
+ elif not PVALS['edate']:
76
+ PVALS['edate'] = arg
77
+ else:
78
+ PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
79
+
80
+ PgDBI.ivaddb_dbname()
81
+
82
+ if(not (PVALS['bdate'] and PVALS['edate']) or not re.match(r'^(daily|monthly|yearly)$', PVALS['group']) or
83
+ not (PVALS['aname'] and PVALS['fname'] and PVALS['wcnd'])):
84
+ pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
85
+ print("Usage: countattmvar -g GroupBy (daily|monthly|yearly) -a AttmName -f FieldName -w WhereCondition [-j JoinName] BeginDate EndDate")
86
+ print(" Group by Daily, Monthly or Yearly is mandatory")
87
+ print(" Set BeginDate and EndDate between '' and '{}'".format(pgrec['bdate'], pgrec['edate']))
88
+ sys.exit(0)
89
+
90
+ if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
91
+ tmpdate = PVALS['bdate']
92
+ PVALS['bdate'] = PVALS['edate']
93
+ PVALS['edate'] = tmpdate
94
+
95
+ if PVALS['jname'] and (PVALS['jname'] == PVALS['aname'] or PVALS['jname'] == "icoreloc"): PVALS['jname'] = None
96
+
97
+ PgLOG.PGLOG['LOGFILE'] = "icoads.log"
98
+ PgLOG.cmdlog("countattmvar {}".format(' '.join(argv)))
99
+
100
+ if not PVALS['oname']:
101
+ PVALS['oname'] = "{}.{}_COUNTS_{}_{}-{}.txt".format(PVALS['aname'], PVALS['fname'],
102
+ PVALS['group'].upper(), PVALS['bdate'], PVALS['edate'])
103
+
104
+ IMMA = open(PVALS['oname'], 'w')
105
+ IMMA.write("{}, {}\n".format(PVALS['group'], PVALS['fname']))
106
+ count_attm_variable(IMMA)
107
+ IMMA.close()
108
+ PgLOG.PgLOG.cmdlog()
109
+ sys.exit(0)
110
+
111
+ #
112
+ # count the variable of a attm daily/month/y/yearly
113
+ #
114
+ def count_attm_variable(IMMA):
115
+
116
+ pcnt = init_periods()
117
+ tcnt = 0
118
+ for pidx in range(pcnt):
119
+ if PVALS['group'] == 'daily':
120
+ acnt = count_daily_attm_variable(PVALS['bpdate'][pidx])
121
+ else:
122
+ acnt = count_period_attm_variable(pidx)
123
+ if not acnt: continue
124
+ IMMA.write("{}, {}\n".format(PVALS['period'][pidx], acnt))
125
+ tcnt += acnt
126
+
127
+ if pcnt > 1:
128
+ IMMA.write("Total, {}\n".format(tcnt))
129
+ PgLOG.pglog("{}.{}: {} total for {} {} periods".format(PVALS['aname'], PVALS['fname'], tcnt, pcnt, PVALS['group']), PgLOG.LOGWRN)
130
+
131
+ #
132
+ # read icoads record from given file name and save them into RDADB
133
+ #
134
+ def count_period_attm_variable(pidx):
135
+
136
+ bdate = PVALS['bpdate'][pidx]
137
+ edate = PVALS['epdate'][pidx]
138
+ btidx = PgIMMA.date2tidx(bdate)
139
+ etidx = PgIMMA.date2tidx(edate)
140
+ tblcnt = etidx-btidx+1
141
+ PgLOG.pglog("Counting {}.{} for {} from IVADDB".format(PVALS['aname'], PVALS['fname'], PVALS['period'][pidx]), PgLOG.WARNLG)
142
+
143
+ # get acount from the first table
144
+ acount = 0
145
+ if tblcnt == 1:
146
+ cnds = ["date BETWEEN '{}' AND '{}' AND ".format(bdate, edate)]
147
+ else:
148
+ cnds = ['']*tblcnt
149
+ cnds[0] = "date >='{}' AND ".format(bdate)
150
+ cnds[tblcnt-1] = "date <= '{}' AND ".format(edate)
151
+ for i in range(tblcnt): acount += count_table_attm_variable(cnds[i], i + btidx)
152
+
153
+ if acount > 0:
154
+ PgLOG.pglog("{}.{}: {} for {} of {}".format(PVALS['aname'], PVALS['fname'], acount, PVALS['wcnd'], PVALS['period'][pidx]), PgLOG.LOGWRN)
155
+ return acount
156
+
157
+ #
158
+ # count IMMA records for given date
159
+ #
160
+ def count_daily_attm_variable(cdate):
161
+
162
+ tidx = PgIMMA.date2tidx(cdate)
163
+ if not tidx: return 0
164
+ PgLOG.pglog("Counting {}.{} for {} from IVADDB".format(PVALS['aname'], PVALS['fname'], cdate), PgLOG.WARNLG)
165
+ acount = count_table_attm_variable("date = '{}' AND ".format(cdate), tidx)
166
+ if acount > 0:
167
+ PgLOG.pglog("{}.{}: {} for {} of ".format(PVALS['aname'], PVALS['fname'], acount, PVALS['wcnd'], cdate), PgLOG.LOGWRN)
168
+ return acount
169
+
170
+ #
171
+ # count IMMA records from one table
172
+ #
173
+ def count_table_attm_variable(dcnd, tidx):
174
+
175
+ if PVALS['jname']:
176
+ jtable = "{}_{}".format(PVALS['jname'], tidx)
177
+ if tidx not in PVALS['jtables']: PVALS['jtables'][tidx] = PgDBI.pgcheck(jtable)
178
+ if not PVALS['jtables'][tidx]: return 0
179
+
180
+ atable = "{}_{}".format(PVALS['aname'], tidx)
181
+ if tidx not in PVALS['atables']: PVALS['atables'][tidx] = PgDBI.pgcheck(atable)
182
+ if not PVALS['atables'][tidx]: return 0
183
+
184
+ if PVALS['aname'] == 'icoreloc':
185
+ if PVALS['jname']:
186
+ table = "{} j, {} n".format(jtable, atable)
187
+ jcnd = "j.iidx = n.iidx AND "
188
+ else:
189
+ table = atable
190
+ jcnd = ""
191
+ else:
192
+ mtable = "icoreloc_{}".format(tidx)
193
+ if PVALS['jname']:
194
+ table = "{}m, {} j. {} n".format(mtable, jtable, atable)
195
+ jcnd = "m.iidx = j.iidx AND m.iidx = n.iidx AND "
196
+ else:
197
+ table = "{} m, {} n".format(mtable, atable)
198
+ cnd = "{} AND m.iidx = n.iidx AND "
199
+
200
+ return PgDBI.pgget(table, "", dcnd + jcnd + PVALS['wcnd'], PgLOG.LGEREX)
201
+
202
+ #
203
+ # initialize the group periods
204
+ #
205
+ def init_periods():
206
+
207
+ bdate = PVALS['bdate']
208
+
209
+ if PVALS['group'] == "yearly":
210
+ dfmt = "YYYY"
211
+ eflg = "Y"
212
+ elif PVALS['group'] == 'monthly':
213
+ dfmt = "YYYY-MM"
214
+ eflg = "M"
215
+ else: # must be daily
216
+ dfmt = "YYYY-MM-DD"
217
+ eflg = ""
218
+
219
+ pcnt = 0
220
+ while True:
221
+ pcnt += 1
222
+ PVALS['bpdate'].append(bdate)
223
+ PVALS['period'].append(PgUtil.format_date(bdate, dfmt))
224
+ edate = PgUtil.enddate(bdate, 0, eflg) if eflg else bdate
225
+ if PgUtil.diffdate(edate, PVALS['edate']) < 0:
226
+ PVALS['epdate'].append(edate)
227
+ bdate = PgUtil.adddate(edate, 0, 0, 1)
228
+ else:
229
+ PVALS['epdate'].append(PVALS['edate'])
230
+ break
231
+
232
+ return pcnt
233
+
234
+ #
235
+ # call main() to start program
236
+ #
237
+ if __name__ == "__main__": main()
@@ -0,0 +1,221 @@
1
+ #!/usr/bin/env python3
2
+ #
3
+ ##################################################################################
4
+ #
5
+ # Title : countsst
6
+ # Author : Zaihua Ji, zji@ucar.edu
7
+ # Date : 01/09/2021
8
+ # 2025-03-04 transferred to package rda_python_icoads from
9
+ # https://github.com/NCAR/rda-icoads.git
10
+ # Purpose : read ICOADS data from IVADDB and count SST records by dail, monthly
11
+ # or yearly
12
+ #
13
+ # Github : https://github.com/NCAR/rda-python-icoads.git
14
+ #
15
+ ##################################################################################
16
+
17
+ import sys
18
+ import os
19
+ import re
20
+ from os import path as op
21
+ from rda_python_common import PgLOG
22
+ from rda_python_common import PgDBI
23
+ from rda_python_common import PgUtil
24
+ from . import PgIMMA
25
+
26
+ PVALS = {
27
+ 'bdate' : None,
28
+ 'edate' : None,
29
+ 'bpdate' : [],
30
+ 'epdate' : [],
31
+ 'period' : [],
32
+ 'group' : None,
33
+ 'oname' : None
34
+ }
35
+
36
+ #
37
+ # main function to run dsarch
38
+ #
39
+ def main():
40
+
41
+ option = None
42
+ argv = sys.argv[1:]
43
+
44
+ for arg in argv:
45
+ if arg == "-b":
46
+ PgLOG.PGLOG['BCKGRND'] = 1
47
+ continue
48
+ ms = re.match(r'-([go])$', arg)
49
+ if ms:
50
+ option = ms.group(1)
51
+ continue
52
+ if re.match(r'^-', arg): PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
53
+ elif option:
54
+ if option == 'g':
55
+ PVALS['group'] = arg
56
+ elif option == 'o':
57
+ PVALS['oname'] = arg
58
+ option = ''
59
+ elif not PVALS['bdate']:
60
+ PVALS['bdate'] = arg
61
+ elif not PVALS['edate']:
62
+ PVALS['edate'] = arg
63
+ else:
64
+ PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
65
+
66
+ PgDBI.ivaddb_dbname()
67
+
68
+ if not (PVALS['bdate'] and PVALS['edate'] and re.match(r'^(daily|monthly|yearly)$', PVALS['group'])):
69
+ pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
70
+ print("Usage: countsst -g GroupBy (daily|monthly|yearly) BeginDate EndDate")
71
+ print(" Group by Daily, Monthly or Yearly is mandatory")
72
+ print(" Set BeginDate and EndDate between '{}' and '{}'".format(pgrec['bdate'], pgrec['edate']))
73
+ sys.exit(0)
74
+
75
+ if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
76
+ tmpdate = PVALS['bdate']
77
+ PVALS['bdate'] = PVALS['edate']
78
+ PVALS['edate'] = tmpdate
79
+
80
+ PgLOG.PGLOG['LOGFILE'] = "icoads.log"
81
+ PgLOG.cmdlog("countsst {}".format(' '.join(argv)))
82
+
83
+ if not PVALS['oname']:
84
+ PVALS['oname'] = "SST_COUNTS_{}_{}-{}.txt".format(PVALS['group'].upper(), PVALS['bdate'], PVALS['edate'])
85
+
86
+ IMMA = open(PVALS['oname'], 'w')
87
+ IMMA.write(PVALS['group'] + ", NOCN, ENH, SST, TOTAL\n")
88
+ count_sst(IMMA)
89
+ IMMA.close()
90
+ PgLOG.cmdlog()
91
+ sys.exit(0)
92
+
93
+ #
94
+ # count the SST values daily/monthly/yearly
95
+ #
96
+ def count_sst(fd):
97
+
98
+ pcnt = init_periods()
99
+ tcnts = [0]*4
100
+ getdaily = 1 if PVALS['group'] == 'daily' else 0
101
+
102
+ for pidx in range(pcnt):
103
+ if getdaily:
104
+ acnts = count_daily_sst(PVALS['bpdate'][pidx])
105
+ else:
106
+ acnts = count_period_sst(pidx)
107
+
108
+ line = PVALS['period'][pidx]
109
+ for i in range(4):
110
+ line += ", {}".format(acnts[i])
111
+ tcnts[i] += acnts[i]
112
+ fd.write(line + "\n")
113
+
114
+ if pcnt > 1:
115
+ line = "Total"
116
+ for i in range(4):
117
+ line += ", {}".format(tcnts[i])
118
+ fd.write(line + "\n")
119
+ PgLOG.pglog("{} total for {} {} periods".format(tcnts[3], pcnt, PVALS['group']), PgLOG.LOGWRN)
120
+
121
+ #
122
+ # read icoads record from given file name and save them into RDADB
123
+ #
124
+ def count_period_sst(pidx):
125
+
126
+ bdate = PVALS['bpdate'][pidx]
127
+ edate = PVALS['epdate'][pidx]
128
+ btidx = PgIMMA.date2tidx(bdate)
129
+ etidx = PgIMMA.date2tidx(edate)
130
+ tblcnt = etidx-btidx+1
131
+ acounts = [0]*4
132
+ acnds = ['']*tblcnt
133
+ mcnds = ['']*tblcnt
134
+
135
+ PgLOG.pglog("Counting SST for {} from IVADDB".format(PVALS['period'][pidx]), PgLOG.WARNLG)
136
+
137
+ if tblcnt == 1:
138
+ acnds[0] = "date BETWEEN '{}' AND '{}'".format(bdate, edate)
139
+ mcnds[0] = "time BETWEEN '{} 00:00:00' AND '{} 23:59:59'".format(bdate, edate)
140
+ else:
141
+ acnds[0] = "date >= '{}'".format(bdate)
142
+ mcnds[0] = "time >= '{} 00:00:00'".format(bdate)
143
+ acnds[tblcnt-1] = "date <= '{}'".format(edate)
144
+ mcnds[tblcnt-1] = "time <= '{} 23:59:59'".format(edate)
145
+
146
+ for i in range(tblcnt): count_table_sst(btidx+i, acounts, acnds[i], mcnds[i])
147
+ if acounts[3]: PgLOG.pglog("{} for {}".format(acounts[3], PVALS['period'][pidx]), PgLOG.LOGWRN)
148
+
149
+ return acounts
150
+
151
+ #
152
+ # count IMMA records for given date
153
+ #
154
+ def count_daily_sst(cdate):
155
+
156
+ tidx = PgIMMA.date2tidx(cdate)
157
+ acounts = [0]*4
158
+
159
+ PgLOG.pglog("Counting SST for {} from IVADDB".format(cdate), PgLOG.WARNLG)
160
+ acnd = "date = '{}'".format(cdate)
161
+ mcnd = "time BETWEEN '{} 00:00:00' AND '{} 23:59:59'".format(cdate, cdate)
162
+ count_table_sst(tidx, acounts, acnd, mcnd)
163
+ if acounts[3]: PgLOG.pglog("{} for {}".format(acounts[3], cdate), PgLOG.LOGWRN)
164
+
165
+ return acounts
166
+
167
+ #
168
+ # count IMMA records from one table
169
+ #
170
+ def count_table_sst(tidx, acounts, acnd, mcnd):
171
+
172
+ table = "icoreloc_{}".format(tidx)
173
+ cnt = PgDBI.pgget(table, "", acnd, PgLOG.LGEREX)
174
+ if not cnt: return acounts
175
+ acounts[3] += cnt
176
+
177
+ table = 'domsdb.idoms_{}'.format(tidx)
178
+ if mcnd: mcnd += ' AND '
179
+ mcnd += "sea_water_temperature IS NOT NULL"
180
+ acounts[2] += PgDBI.pgget(table, "", mcnd, PgLOG.LGEREX)
181
+ acnd = mcnd + " AND sea_water_temperature_quality = 0"
182
+ acounts[1] += PgDBI.pgget(table, "", acnd, PgLOG.LGEREX)
183
+ acnd = mcnd + " AND sea_water_temperature_depth > 0"
184
+ acounts[0] += PgDBI.pgget(table, "", acnd, PgLOG.LGEREX)
185
+
186
+ return acounts
187
+
188
+ #
189
+ # initialize period list
190
+ #
191
+ def init_periods():
192
+
193
+ bdate = PVALS['bdate']
194
+ if PVALS['group'] == "yearly":
195
+ dfmt = "YYYY"
196
+ eflg = "Y"
197
+ elif PVALS['group'] == 'monthly':
198
+ dfmt = "YYYY-MM"
199
+ eflg = "M"
200
+ else: # must be daily
201
+ dfmt = "YYYY-MM-DD"
202
+ eflg = ""
203
+ pcnt = 0
204
+ while True:
205
+ pcnt += 1
206
+ PVALS['bpdate'].append(bdate)
207
+ PVALS['period'].append(PgUtil.format_date(bdate, dfmt))
208
+ edate = PgUtil.enddate(bdate, 0, eflg) if eflg else bdate
209
+ if PgUtil.diffdate(PVALS['edate'], edate) > 0:
210
+ PVALS['epdate'].append(edate)
211
+ bdate = PgUtil.adddate(edate, 0, 0, 1)
212
+ else:
213
+ PVALS['epdate'].append(PVALS['edate'])
214
+ break
215
+
216
+ return pcnt
217
+
218
+ #
219
+ # call main() to start program
220
+ #
221
+ if __name__ == "__main__": main()
@@ -33,8 +33,10 @@ PVALS = {
33
33
  def main():
34
34
 
35
35
  addinventory = leaduid = chkexist = 0
36
+ rn3 = -1
36
37
  argv = sys.argv[1:]
37
-
38
+
39
+ option = None
38
40
  for arg in argv:
39
41
  if arg == "-b":
40
42
  PgLOG.PGLOG['BCKGRND'] = 1
@@ -44,27 +46,32 @@ def main():
44
46
  leaduid = 1
45
47
  elif arg == "-e":
46
48
  chkexist = 1
49
+ elif arg == "-r":
50
+ option = 'r'
47
51
  elif arg == "-i":
48
52
  addinventory = 1
49
53
  elif re.match(r'^-', arg):
50
54
  PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
55
+ elif option == 'r':
56
+ rn3 = int(arg)
51
57
  else:
52
58
  PVALS['files'].append(arg)
53
59
 
54
60
  if not PVALS['files']:
55
- print("Usage: fillicoads [-a] [-e] [-i] [-u] FileNameList")
61
+ print("Usage: fillicoads [-a] [-e] [-i] [-r] [-u] FileNameList")
56
62
  print(" At least one file name needs to fill icoads data into Postgres Server")
57
63
  print(" Option -a: add all attms, including multi-line ones, such as IVAD and REANQC")
58
64
  print(" Option -i: add daily counting records into inventory table")
65
+ print(" Option -r: last digit of IMMA release number")
59
66
  print(" Option -u: standalone attachment records with leading 6-character UID")
60
67
  print(" Option -e: check existing record before adding attm")
61
68
  sys.exit(0)
62
69
 
63
70
  PgLOG.PGLOG['LOGFILE'] = "icoads.log"
64
- PgDBI.ivaddb_dbname()
65
-
71
+ PgDBI.set_scname(dbname = 'ivaddb', scname = 'ivaddb1', lnname = 'ivaddb', dbhost = PgLOG.PGLOG['PMISCHOST'])
72
+
66
73
  PgLOG.cmdlog("fillicoads {}".format(' '.join(argv)))
67
- PgIMMA.init_current_indices(leaduid, chkexist)
74
+ PgIMMA.init_current_indices(leaduid, chkexist, rn3)
68
75
  PVALS['names'] = '/'.join(PgIMMA.IMMA_NAMES)
69
76
  fill_imma_data(addinventory)
70
77
  PgLOG.cmdlog()
@@ -103,14 +110,14 @@ def process_imma_file(fname, addinventory):
103
110
  idate = cdate = PgIMMA.get_imma_date(line)
104
111
  if cdate:
105
112
  PgIMMA.init_indices_for_date(cdate, iname)
106
- records = PgIMMA.get_imma_records(line, cdate, records)
113
+ records = PgIMMA.get_imma_records(cdate, line, records)
107
114
  break
108
115
  line = IMMA.readline()
109
116
 
110
117
  line = IMMA.readline()
111
118
  while line:
112
119
  if PVALS['uatti'] and line[0:2] == PVALS['uatti']:
113
- records = PgIMMA.get_imma_multiple_records(line, records)
120
+ records = PgIMMA.get_imma_multiple_records(cdate, line, records)
114
121
  else:
115
122
  idate = PgIMMA.get_imma_date(line)
116
123
  if idate:
@@ -120,7 +127,7 @@ def process_imma_file(fname, addinventory):
120
127
  records = {}
121
128
  cdate = idate
122
129
  PgIMMA.init_indices_for_date(cdate, iname)
123
- records = PgIMMA.get_imma_records(line, idate, records)
130
+ records = PgIMMA.get_imma_records(idate, line, records)
124
131
  line = IMMA.readline()
125
132
 
126
133
  IMMA.close()