rda-python-dsupdt 2.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rda_python_dsupdt/PgUpdt.py +1921 -0
- rda_python_dsupdt/__init__.py +1 -0
- rda_python_dsupdt/ds_updt.py +2454 -0
- rda_python_dsupdt/dsupdt.py +2134 -0
- rda_python_dsupdt/dsupdt.usg +1774 -0
- rda_python_dsupdt/pg_updt.py +1652 -0
- rda_python_dsupdt-2.0.4.dist-info/METADATA +18 -0
- rda_python_dsupdt-2.0.4.dist-info/RECORD +12 -0
- rda_python_dsupdt-2.0.4.dist-info/WHEEL +5 -0
- rda_python_dsupdt-2.0.4.dist-info/entry_points.txt +2 -0
- rda_python_dsupdt-2.0.4.dist-info/licenses/LICENSE +21 -0
- rda_python_dsupdt-2.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1921 @@
|
|
|
1
|
+
#
|
|
2
|
+
###############################################################################
|
|
3
|
+
#
|
|
4
|
+
# Title : PgUpdt.py
|
|
5
|
+
# Author : Zaihua Ji, zji@ucar.edu
|
|
6
|
+
# Date : 09/23/2020
|
|
7
|
+
# 2025-02-07 transferred to package rda_python_dsupdt from
|
|
8
|
+
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
|
+
# Purpose : python library module to help rountinely updates of new data
|
|
10
|
+
# for one or multiple datasets
|
|
11
|
+
#
|
|
12
|
+
# Github : https://github.com/NCAR/rda-python-dsupdt.git
|
|
13
|
+
#
|
|
14
|
+
###############################################################################
|
|
15
|
+
#
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import time
|
|
19
|
+
from os import path as op
|
|
20
|
+
from rda_python_common import PgLOG
|
|
21
|
+
from rda_python_common import PgSIG
|
|
22
|
+
from rda_python_common import PgLock
|
|
23
|
+
from rda_python_common import PgCMD
|
|
24
|
+
from rda_python_common import PgOPT
|
|
25
|
+
from rda_python_common import PgFile
|
|
26
|
+
from rda_python_common import PgUtil
|
|
27
|
+
from rda_python_common import PgDBI
|
|
28
|
+
|
|
29
|
+
CORDERS = ()
|
|
30
|
+
|
|
31
|
+
PgOPT.OPTS = {
|
|
32
|
+
'DR' : [0x00010, 'DownloadRemote',2],
|
|
33
|
+
'BL' : [0x00020, 'BuildLocal', 2],
|
|
34
|
+
'PB' : [0x00030, 'ProcessBoth', 2], # DR & BL
|
|
35
|
+
'AF' : [0x00040, 'ArchiveFile', 2],
|
|
36
|
+
'CF' : [0x00080, 'CleanFile', 2],
|
|
37
|
+
'UF' : [0x000F0, 'UpdateFile', 2], # DR & BL & AF & CF
|
|
38
|
+
'CU' : [0x00200, 'CheckUpdate', 0],
|
|
39
|
+
'GC' : [0x00400, 'GetControl', 0],
|
|
40
|
+
'GL' : [0x00800, 'GetLocalFile', 0],
|
|
41
|
+
'GR' : [0x01000, 'GetRemoteFile', 0],
|
|
42
|
+
'GA' : [0x01C00, 'GetALL', 0], # GC & GL & GR
|
|
43
|
+
'SC' : [0x02000, 'SetControl', 1],
|
|
44
|
+
'SL' : [0x04000, 'SetLocalFile', 1],
|
|
45
|
+
'SR' : [0x08000, 'SetRemoteFile', 1],
|
|
46
|
+
'SA' : [0x0E000, 'SetALL', 4], # SC & SL & SR
|
|
47
|
+
'DL' : [0x20000, 'Delete', 1],
|
|
48
|
+
'UL' : [0x40000, 'UnLock', 1],
|
|
49
|
+
|
|
50
|
+
'AW' : [0, 'AnyWhere'],
|
|
51
|
+
'BG' : [0, 'BackGround'],
|
|
52
|
+
'CA' : [0, 'CheckAll'],
|
|
53
|
+
'CN' : [0, 'CheckNew'],
|
|
54
|
+
'CP' : [0, 'CurrrentPeriod'],
|
|
55
|
+
'EE' : [0, 'ErrorEmail'], # send email when error happens only
|
|
56
|
+
'FO' : [0, 'FormatOutput'],
|
|
57
|
+
'FU' : [0, 'FutureUpdate'],
|
|
58
|
+
'GZ' : [0, 'GMTZone'],
|
|
59
|
+
'HU' : [0, 'HourlyUpdate'],
|
|
60
|
+
'IE' : [0, 'IgnoreError'],
|
|
61
|
+
'KR' : [0, 'KeepRemote'],
|
|
62
|
+
'KS' : [0, 'KeepServer'],
|
|
63
|
+
'LO' : [0, 'LogOn'],
|
|
64
|
+
'MD' : [0, 'PgDataset'],
|
|
65
|
+
'MO' : [0, 'MissedOnly'],
|
|
66
|
+
'MU' : [0, 'MultipleUpdate'],
|
|
67
|
+
'NC' : [0, 'NewControl'],
|
|
68
|
+
'NE' : [0, 'NoEmail'],
|
|
69
|
+
'NL' : [0, 'NewLocfile'],
|
|
70
|
+
'NY' : [0, 'NoLeapYear'],
|
|
71
|
+
'QE' : [0, 'QuitError'],
|
|
72
|
+
'RA' : [0, 'RetryArchive'],
|
|
73
|
+
'RD' : [0, 'RetryDownload'],
|
|
74
|
+
'RE' : [0, 'ResetEndTime'],
|
|
75
|
+
'RO' : [0, 'ResetOrder'],
|
|
76
|
+
'SE' : [0, 'SummaryEmail'], # send summary email only
|
|
77
|
+
'UB' : [0, 'UseBeginTime'],
|
|
78
|
+
'UT' : [0, 'UpdateTime'],
|
|
79
|
+
|
|
80
|
+
'AO' : [1, 'ActOption', 1], # default to <!>
|
|
81
|
+
'CD' : [1, 'CurrentDate', 256], # used this instead of curdate()
|
|
82
|
+
'CH' : [1, 'CurrentHour', 16], # used this instead of (localtime)[2]
|
|
83
|
+
'DS' : [1, 'Dataset', 0],
|
|
84
|
+
'DV' : [1, 'Divider', 1], # default to <:>
|
|
85
|
+
'ES' : [1, 'EqualSign', 1], # default to <=>
|
|
86
|
+
'FN' : [1, 'FieldNames', 0],
|
|
87
|
+
'LN' : [1, 'LoginName', 1],
|
|
88
|
+
'OF' : [1, 'OutputFile', 0],
|
|
89
|
+
'ON' : [1, 'OrderNames', 0],
|
|
90
|
+
'PL' : [1, 'ProcessLimit', 17],
|
|
91
|
+
'VS' : [1, 'ValidSize', 17], # default to PgLOG.PGLOG['MINSIZE']
|
|
92
|
+
|
|
93
|
+
'AN' : [2, 'ActionName', 1],
|
|
94
|
+
'AT' : [2, 'AgeTime', 1],
|
|
95
|
+
'BC' : [2, 'BuildCommand', 1],
|
|
96
|
+
'BP' : [2, 'BatchProcess', 0, ''],
|
|
97
|
+
'BT' : [2, 'BeginTime', 1],
|
|
98
|
+
'CC' : [2, 'CarbonCopy', 0],
|
|
99
|
+
'CI' : [2, 'ControlIndex', 16],
|
|
100
|
+
'CL' : [2, 'CleanCommand', 1],
|
|
101
|
+
'CO' : [2, "ControlOffset", 1],
|
|
102
|
+
'CT' : [2, 'ControlTime', 32+356],
|
|
103
|
+
'DB' : [2, 'Debug', 0],
|
|
104
|
+
'DC' : [2, 'DownloadCommand', 1],
|
|
105
|
+
'DE' : [2, 'Description', 64],
|
|
106
|
+
'DO' : [2, 'DownloadOrder', 16],
|
|
107
|
+
'DT' : [2, 'DataTime', 1+32+256],
|
|
108
|
+
'EC' : [2, 'ErrorControl', 1, "NIQ"],
|
|
109
|
+
'ED' : [2, 'EndDate', 257],
|
|
110
|
+
'EH' : [2, 'EndHour', 33],
|
|
111
|
+
'EP' : [2, 'EndPeriod', 1],
|
|
112
|
+
'ET' : [2, 'EndTime', 33],
|
|
113
|
+
'FA' : [2, 'FileArchived', 0],
|
|
114
|
+
'FQ' : [2, 'Frequency', 1],
|
|
115
|
+
'GP' : [2, 'GenericPattern', 0],
|
|
116
|
+
'HN' : [2, "HostName", 1],
|
|
117
|
+
'HO' : [2, 'HourOffset', 17],
|
|
118
|
+
'ID' : [2, 'ControlID', 0],
|
|
119
|
+
'IF' : [2, 'InputFile', 0],
|
|
120
|
+
'KF' : [2, 'KeepFile', 1, "NRSB"],
|
|
121
|
+
'LF' : [2, 'LocalFile', 0],
|
|
122
|
+
'LI' : [2, 'LocalIndex', 17],
|
|
123
|
+
'MC' : [2, 'EMailControl', 1, "ASNEB"],
|
|
124
|
+
'MR' : [2, 'MissRemote', 128, "NY"],
|
|
125
|
+
'DI' : [2, 'DueInterval', 1],
|
|
126
|
+
'OP' : [2, 'Options', 1],
|
|
127
|
+
'PD' : [2, 'PatternDelimiter', 2], # pattern delimiters, default to ["<", ">"]
|
|
128
|
+
'PI' : [2, 'ParentIndex', 17],
|
|
129
|
+
'PR' : [2, 'ProcessRemote', 1],
|
|
130
|
+
'QS' : [2, 'QSubOptions', 0],
|
|
131
|
+
'RF' : [2, 'RemoteFile', 0],
|
|
132
|
+
'RI' : [2, 'RetryInterval', 1],
|
|
133
|
+
'SB' : [2, 'SBatchOptions', 1],
|
|
134
|
+
'SF' : [2, 'ServerFile', 0],
|
|
135
|
+
'SN' : [2, 'Specialist', 1],
|
|
136
|
+
'TI' : [2, 'TimeInterval', 1],
|
|
137
|
+
'UC' : [2, 'UpdateControl', 1],
|
|
138
|
+
'VI' : [2, 'ValidInterval', 1],
|
|
139
|
+
'WD' : [2, 'WorkDir', 1],
|
|
140
|
+
'XC' : [2, 'ExecuteCommand', 1],
|
|
141
|
+
'XO' : [2, 'ExecOrder', 16],
|
|
142
|
+
}
|
|
143
|
+
PgOPT.ALIAS = {
|
|
144
|
+
'AN' : ['Action', "AC"],
|
|
145
|
+
'AT' : ['FileAge', "FileAgeTime"],
|
|
146
|
+
'BC' : ['BuildCmd'],
|
|
147
|
+
'BG' : ['b'],
|
|
148
|
+
'BL' : ['BuildLocalfile'],
|
|
149
|
+
'BP' : ['d', 'DelayedMode'],
|
|
150
|
+
'BT' : ['IT', 'InitialTime'],
|
|
151
|
+
'CI' : ['UpdateControlIndex'],
|
|
152
|
+
'CL' : ['CleanFile'],
|
|
153
|
+
'CN' : ['CheckNewFile'],
|
|
154
|
+
'DC' : ['Command', 'Download'],
|
|
155
|
+
'DE' : ['Desc', 'Note', 'FileDesc', 'FileDescription'],
|
|
156
|
+
'DI' : ['NextDue'],
|
|
157
|
+
'DL' : ['RM', 'Remove'],
|
|
158
|
+
'DR' : ['DownloadRemoteFile'],
|
|
159
|
+
'DS' : ['Dsid', 'DatasetID'],
|
|
160
|
+
'DV' : ['Delimiter', 'Separator'],
|
|
161
|
+
'ED' : ['UpdateEndDate'],
|
|
162
|
+
'EH' : ['UpdateEndHour'],
|
|
163
|
+
'EP' : ['EndPeriodDay'],
|
|
164
|
+
'FA' : ['SF', 'WF', 'QF'],
|
|
165
|
+
'FQ' : ['UpdateFrequency'],
|
|
166
|
+
'FU' : ["ForceUpdate"],
|
|
167
|
+
'GC' : ['GetUpdateControl'],
|
|
168
|
+
'GL' : ['GetLocal'],
|
|
169
|
+
'GN' : ['GroupID'],
|
|
170
|
+
'GP' : ['GeneralPattern'],
|
|
171
|
+
'GR' : ['GetRemote'],
|
|
172
|
+
'GZ' : ['GMT', 'GreenwichZone', 'UTC'],
|
|
173
|
+
'HN' : ['HostMachine'],
|
|
174
|
+
'KR' : ['KeepRemoteFile'],
|
|
175
|
+
'KS' : ['KeepServerFile'],
|
|
176
|
+
'LF' : ['LocalFileIndex'],
|
|
177
|
+
'LI' : ['LocIndex', "UpdateIndex"],
|
|
178
|
+
'LO' : ['LoggingOn'],
|
|
179
|
+
'OP' : ['DsarchOption'],
|
|
180
|
+
'NC' : ['NewUpdateControl'],
|
|
181
|
+
'NL' : ['NewLocalFile'],
|
|
182
|
+
'PD' : ['TD', 'TemporalDelimiter'],
|
|
183
|
+
'QE' : ['QuitOnError'],
|
|
184
|
+
'QS' : ['PBSOptions'],
|
|
185
|
+
'RD' : ['Redownlaod'],
|
|
186
|
+
'RO' : ['Reorder'],
|
|
187
|
+
'SB' : ['SlurmOptions'],
|
|
188
|
+
'SC' : ['SetUpdateControl'],
|
|
189
|
+
'SL' : ['SetLocal'],
|
|
190
|
+
'SN' : ['SpecialistName'],
|
|
191
|
+
'SR' : ['SetRemote'],
|
|
192
|
+
'TI' : ['Interval'],
|
|
193
|
+
'UL' : ["UnLockUpdate"],
|
|
194
|
+
'XC' : ['ExecCmd'],
|
|
195
|
+
'XO' : ['ExecuteOrder']
|
|
196
|
+
}
|
|
197
|
+
#
|
|
198
|
+
# single letter short names for option 'FN' (Field Names) to retrieve info
|
|
199
|
+
# from RDADB; only the fields can be manipulated by this application are listed
|
|
200
|
+
#
|
|
201
|
+
# SHORTNM KEYS(PgOPT.OPTS) DBFIELD
|
|
202
|
+
PgOPT.TBLHASH['dlupdt'] = { # condition flag, 0-int, 1-string, -1-exclude
|
|
203
|
+
'L' : ['LI', "lindex", 0],
|
|
204
|
+
'F' : ['LF', "locfile", 1],
|
|
205
|
+
'A' : ['AN', "action", 1], # dsarch action
|
|
206
|
+
'I' : ['CI', "cindex", 0],
|
|
207
|
+
'U' : ['FA', "archfile", 1],
|
|
208
|
+
'X' : ['XO', "execorder", 1],
|
|
209
|
+
'S' : ['SN', "specialist", 1],
|
|
210
|
+
'M' : ['MR', "missremote", 1],
|
|
211
|
+
'W' : ['WD', "workdir", 1],
|
|
212
|
+
'O' : ['OP', "options", 1],
|
|
213
|
+
'C' : ['DC', "download", 1],
|
|
214
|
+
'Q' : ['FQ', "frequency", 1],
|
|
215
|
+
'E' : ['EP', "endperiod", 0],
|
|
216
|
+
'J' : ['ED', "enddate", 1],
|
|
217
|
+
'K' : ['EH', "endhour", 0],
|
|
218
|
+
'N' : ['DI', "nextdue", 1],
|
|
219
|
+
'V' : ['VI', "validint", 1],
|
|
220
|
+
'T' : ['AT', "agetime", 1],
|
|
221
|
+
'R' : ['PR', "processremote", 1],
|
|
222
|
+
'B' : ['BC', "buildcmd", 1],
|
|
223
|
+
'Z' : ['CL', "cleancmd", 1],
|
|
224
|
+
'D' : ['DE', "note", 1],
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
PgOPT.TBLHASH['drupdt'] = {
|
|
228
|
+
'L' : ['LI', "lindex", 0], # same as dlupdt.lindex
|
|
229
|
+
'F' : ['RF', "remotefile", 1],
|
|
230
|
+
'D' : ['DO', "dindex", 0],
|
|
231
|
+
'S' : ['SF', "serverfile", 1],
|
|
232
|
+
'C' : ['DC', "download", 1],
|
|
233
|
+
'B' : ['BT', "begintime", 1],
|
|
234
|
+
'E' : ['ET', "endtime", 1],
|
|
235
|
+
'T' : ['TI', "tinterval", 1],
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
PgOPT.TBLHASH['dcupdt'] = {
|
|
239
|
+
'C' : ['CI', "cindex", 0],
|
|
240
|
+
'L' : ['ID', "cntlid", 1],
|
|
241
|
+
'N' : ['SN', "specialist", 1],
|
|
242
|
+
'P' : ['PI', "pindex", 0], # if not 0, refer to another dcupdt.cindex
|
|
243
|
+
'A' : ['AN', "action", 1], # dsupdt action
|
|
244
|
+
'F' : ['FQ', "frequency", 1],
|
|
245
|
+
'O' : ['CO', "cntloffset", 1],
|
|
246
|
+
'T' : ['CT', "cntltime", 1],
|
|
247
|
+
'R' : ['RI', "retryint", 1],
|
|
248
|
+
'V' : ['VI', "validint", 1],
|
|
249
|
+
'U' : ['UC', "updtcntl", 1],
|
|
250
|
+
'J' : ['MC', "emailcntl", 1],
|
|
251
|
+
'E' : ['EC', "errorcntl", 1],
|
|
252
|
+
'K' : ['KF', "keepfile", 1],
|
|
253
|
+
'Z' : ['HO', "houroffset", 1],
|
|
254
|
+
'D' : ['DT', "datatime", 1],
|
|
255
|
+
'H' : ['HN', "hostname", 1],
|
|
256
|
+
'Q' : ['QS', "qoptions", 1],
|
|
257
|
+
'Y' : ['CC', "emails", 1],
|
|
258
|
+
'X' : ['XC', "execcmd", 1],
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
# global info to be used by the whole application
|
|
262
|
+
PgOPT.PGOPT['updated'] = 0
|
|
263
|
+
PgOPT.PGOPT['AUTODS'] = 0
|
|
264
|
+
PgOPT.PGOPT['CNTLACTS'] = PgOPT.OPTS['UF'][0]|PgOPT.OPTS['CU'][0]
|
|
265
|
+
PgOPT.PGOPT['UPDTACTS'] = "AF|BL|CF|CU|DR|PB|UF"
|
|
266
|
+
PgOPT.PGOPT['ARCHACTS'] = "AW|AS|AQ"
|
|
267
|
+
PgOPT.PGOPT['DTIMES'] = {}
|
|
268
|
+
PgOPT.PGOPT['UCNTL'] = {}
|
|
269
|
+
#default fields for getting info
|
|
270
|
+
PgOPT.PGOPT['dlupdt'] = "LFAXIUCOQJNVWRZ"
|
|
271
|
+
PgOPT.PGOPT['drupdt'] = "LFDSCBET"
|
|
272
|
+
PgOPT.PGOPT['dcupdt'] = "CLNPAFOTRVUJEKZ"
|
|
273
|
+
#all fields for getting info
|
|
274
|
+
PgOPT.PGOPT['dlall'] = "LFAXIUCOQEJKNVTWMRBZSD"
|
|
275
|
+
PgOPT.PGOPT['drall'] = PgOPT.PGOPT['drupdt']
|
|
276
|
+
PgOPT.PGOPT['dcall'] = "CLNPAFOTRVUJEKZDHSQYX"
|
|
277
|
+
# remote file download status
|
|
278
|
+
# 0 error download, but continue for further download
|
|
279
|
+
# 1 successful full/partial download, continue for build local files
|
|
280
|
+
# < 0 error download, stop
|
|
281
|
+
PgOPT.PGOPT['rstat'] = 1 # default to successful download
|
|
282
|
+
# counts
|
|
283
|
+
PgOPT.PGOPT['PCNT'] = 1
|
|
284
|
+
PgOPT.PGOPT['vcnt'] = PgOPT.PGOPT['rcnt'] = PgOPT.PGOPT['dcnt'] = PgOPT.PGOPT['lcnt'] = 0
|
|
285
|
+
PgOPT.PGOPT['bcnt'] = PgOPT.PGOPT['acnt'] = PgOPT.PGOPT['mcnt'] = 0
|
|
286
|
+
PgOPT.PGOPT['ucnt'] = PgOPT.PGOPT['upcnt'] = PgOPT.PGOPT['ubcnt'] = PgOPT.PGOPT['uhcnt'] = 0
|
|
287
|
+
PgOPT.PGOPT['uscnt'] = PgOPT.PGOPT['qbcnt'] = PgOPT.PGOPT['qdcnt'] = 0
|
|
288
|
+
PgOPT.PGOPT['uwcnt'] = PgOPT.PGOPT['udcnt'] = PgOPT.PGOPT['uncnt'] = PgOPT.PGOPT['rdcnt'] = 0
|
|
289
|
+
PgOPT.PGOPT['lindex'] = 0 # the current lindex is under updating
|
|
290
|
+
|
|
291
|
+
WSLOWS = {
|
|
292
|
+
'nomads.ncep.noaa.gov' : 8
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
# set default parameters
|
|
296
|
+
PgOPT.params['PD'] = ["<" , ">"] # temporal pattern delimiters
|
|
297
|
+
PgOPT.params['PL'] = 1 # max number of child processes allowed
|
|
298
|
+
|
|
299
|
+
#
|
|
300
|
+
# get file contion
|
|
301
|
+
#
|
|
302
|
+
def file_condition(tname, include = None, exclude = None, nodsid = 0):
|
|
303
|
+
|
|
304
|
+
condition = ""
|
|
305
|
+
hash = PgOPT.TBLHASH[tname]
|
|
306
|
+
noand = 1 if nodsid else 0
|
|
307
|
+
if not hash: PgLOG.pglog(tname + ": not defined in PgOPT.TBLHASH", PgOPT.PGOPT['extlog'])
|
|
308
|
+
for key in hash:
|
|
309
|
+
if include and include.find(key) < 0: continue
|
|
310
|
+
if exclude and exclude.find(key) > -1: continue
|
|
311
|
+
type = hash[key][2]
|
|
312
|
+
if type < 0: continue # exclude
|
|
313
|
+
opt = hash[key][0]
|
|
314
|
+
if opt not in PgOPT.params: continue
|
|
315
|
+
fld = hash[key][1]
|
|
316
|
+
condition += PgDBI.get_field_condition(fld, PgOPT.params[opt], type, noand)
|
|
317
|
+
noand = 0
|
|
318
|
+
|
|
319
|
+
if not nodsid:
|
|
320
|
+
condition = "dsid = '{}'{}".format(PgOPT.params['DS'], condition)
|
|
321
|
+
|
|
322
|
+
return condition
|
|
323
|
+
|
|
324
|
+
#
|
|
325
|
+
# check if enough information entered on command line and/or input file
|
|
326
|
+
# for given action(s)
|
|
327
|
+
#
|
|
328
|
+
def check_enough_options(cact, acts):
|
|
329
|
+
|
|
330
|
+
errmsg = [
|
|
331
|
+
"Miss dataset number per -DS(-Dataset)",
|
|
332
|
+
"Miss local file names per -LF(-LocalFile)",
|
|
333
|
+
"Miss remote file names per -RF(-RemoteFile)",
|
|
334
|
+
"Miss local Index per -LI(-LocalIndex)",
|
|
335
|
+
"Miss Control Index per -CI(-ControlIndex)",
|
|
336
|
+
"Process one Update Control Index at a time",
|
|
337
|
+
]
|
|
338
|
+
erridx = -1
|
|
339
|
+
lcnt = ccnt = 0
|
|
340
|
+
if 'LI' in PgOPT.params: lcnt = validate_lindices(cact)
|
|
341
|
+
if 'CI' in PgOPT.params or 'ID' in PgOPT.params: ccnt = validate_cindices(cact)
|
|
342
|
+
if PgOPT.OPTS[cact][2] == 1:
|
|
343
|
+
if acts&PgOPT.OPTS['SC'][0]:
|
|
344
|
+
if 'CI' not in PgOPT.params: erridx = 4
|
|
345
|
+
elif cact == 'DL' or cact == 'UL':
|
|
346
|
+
if not ('LI' in PgOPT.params or 'CI' in PgOPT.params): erridx = 3
|
|
347
|
+
elif 'LI' not in PgOPT.params:
|
|
348
|
+
erridx = 3
|
|
349
|
+
elif acts&PgOPT.OPTS['SR'][0] and 'RF' not in PgOPT.params:
|
|
350
|
+
erridx = 2
|
|
351
|
+
|
|
352
|
+
if erridx < 0:
|
|
353
|
+
if (lcnt + ccnt) > 0:
|
|
354
|
+
if 'DS' not in PgOPT.params:
|
|
355
|
+
erridx = 0
|
|
356
|
+
elif lcnt > 0 and cact == 'SL' and 'LF' not in PgOPT.params:
|
|
357
|
+
erridx = 1
|
|
358
|
+
elif PgOPT.OPTS[cact][2] == 2:
|
|
359
|
+
if 'CI' in PgOPT.params and len(PgOPT.params['CI']) > 1:
|
|
360
|
+
erridx = 5
|
|
361
|
+
|
|
362
|
+
if erridx >= 0: PgOPT.action_error(errmsg[erridx], cact)
|
|
363
|
+
|
|
364
|
+
PgOPT.set_uid("dsupdt") # set uid before any action
|
|
365
|
+
|
|
366
|
+
if 'VS' in PgOPT.params: # minimal size for a file to be valid for archive
|
|
367
|
+
PgLOG.PGLOG['MINSIZE'] = int(PgOPT.params['VS'])
|
|
368
|
+
|
|
369
|
+
if 'BP' in PgOPT.params:
|
|
370
|
+
if 'PL' in PgOPT.params: PgOPT.params['PL'] = 1
|
|
371
|
+
if 'CI' in PgOPT.params:
|
|
372
|
+
oidx = PgOPT.params['CI'][0]
|
|
373
|
+
otype = 'C'
|
|
374
|
+
elif 'LI' in PgOPT.params:
|
|
375
|
+
oidx = PgOPT.params['LI'][0]
|
|
376
|
+
otype = 'L'
|
|
377
|
+
else:
|
|
378
|
+
oidx = 0
|
|
379
|
+
otype = ''
|
|
380
|
+
|
|
381
|
+
# set command line Batch options
|
|
382
|
+
PgCMD.set_batch_options(PgOPT.params, 2, 1)
|
|
383
|
+
PgCMD.init_dscheck(oidx, otype, "dsupdt", get_dsupdt_dataset(),
|
|
384
|
+
cact, "" if 'AW' in PgOPT.params else PgLOG.PGLOG['CURDIR'], PgOPT.params['LN'],
|
|
385
|
+
PgOPT.params['BP'], PgOPT.PGOPT['extlog'])
|
|
386
|
+
|
|
387
|
+
if 'NY' in PgOPT.params: PgLOG.PGLOG['NOLEAP'] = 1
|
|
388
|
+
if 'NE' in PgOPT.params:
|
|
389
|
+
PgLOG.PGLOG['LOGMASK'] &= ~PgLOG.EMLALL # turn off all email acts
|
|
390
|
+
else:
|
|
391
|
+
if 'SE' in PgOPT.params: PgOPT.PGOPT['emllog'] |= PgLOG.EMEROL
|
|
392
|
+
if 'CC' in PgOPT.params and (PgOPT.PGOPT['ACTS']&PgOPT.OPTS['SC'][2]) == 2: PgLOG.add_carbon_copy(PgOPT.params['CC'])
|
|
393
|
+
|
|
394
|
+
if PgOPT.PGOPT['ACTS']&PgOPT.OPTS['UF'][0]:
|
|
395
|
+
plimit = PgOPT.params['PL'] if 'PL' in PgOPT.params else 1
|
|
396
|
+
logon = PgOPT.params['LO'] if 'LO' in PgOPT.params else 1
|
|
397
|
+
PgSIG.start_none_daemon('dsupdt', PgOPT.PGOPT['CACT'], PgOPT.params['LN'], plimit, 120, logon)
|
|
398
|
+
else:
|
|
399
|
+
PgSIG.start_none_daemon('dsupdt', PgOPT.PGOPT['CACT'], PgOPT.params['LN'], 1, 120, 1)
|
|
400
|
+
|
|
401
|
+
if PgSIG.PGSIG['MPROC'] > 1:
|
|
402
|
+
PgOPT.PGOPT['emllog'] |= PgLOG.FRCLOG
|
|
403
|
+
PgOPT.PGOPT['wrnlog'] |= PgLOG.FRCLOG
|
|
404
|
+
|
|
405
|
+
#
|
|
406
|
+
# get the associated dataset id
|
|
407
|
+
#
|
|
408
|
+
def get_dsupdt_dataset():
|
|
409
|
+
|
|
410
|
+
if 'DS' in PgOPT.params: return PgOPT.params['DS']
|
|
411
|
+
if 'CI' in PgOPT.params and PgOPT.params['CI'][0]:
|
|
412
|
+
pgrec = PgDBI.pgget("dcupdt", "dsid", "cindex = {}".format(PgOPT.params['CI'][0]), PgOPT.PGOPT['extlog'])
|
|
413
|
+
if pgrec: return pgrec['dsid']
|
|
414
|
+
|
|
415
|
+
if 'LI' in PgOPT.params and PgOPT.params['LI'][0]:
|
|
416
|
+
pgrec = PgDBI.pgget("dlupdt", "dsid", "lindex = {}".format(PgOPT.params['LI'][0]), PgOPT.PGOPT['extlog'])
|
|
417
|
+
if pgrec: return pgrec['dsid']
|
|
418
|
+
|
|
419
|
+
return None
|
|
420
|
+
|
|
421
|
+
#
|
|
422
|
+
# replace the temoral patterns in given fname with date/hour
|
|
423
|
+
#
|
|
424
|
+
# return pattern array only if not date
|
|
425
|
+
#
|
|
426
|
+
def replace_pattern(fname, date, hour = None, intv = None, limit = 0, bdate = None, bhour = None):
|
|
427
|
+
|
|
428
|
+
if not fname: return None
|
|
429
|
+
if date and not isinstance(date, str): date = str(date)
|
|
430
|
+
if bdate and not isinstance(bdate, str): bdate = str(bdate)
|
|
431
|
+
seps = PgOPT.params['PD']
|
|
432
|
+
match = r"[^{}]+".format(seps[1])
|
|
433
|
+
patterns = re.findall(r'{}([^{}]+){}'.format(seps[0], seps[1], seps[1]), fname)
|
|
434
|
+
pcnt = len(patterns)
|
|
435
|
+
if pcnt == 0: return fname # return original name if no pattern
|
|
436
|
+
if limit and pcnt > limit: pcnt = limit
|
|
437
|
+
mps = {'b' : r'^B(.+)B$', 'c': r'^C(.+)C$', 'd' : r'(\d+)$', 'm' : r'^M([NC])M$',
|
|
438
|
+
'n' : r'^N(H+|D+)N$', 'p' : r'^P(\d+)$', 's' : r'^S[\d:]+S$', 'w' : r'^W(.+)W$'}
|
|
439
|
+
for i in range(pcnt):
|
|
440
|
+
pattern = patterns[i]
|
|
441
|
+
replace = "{}{}{}".format(seps[0], pattern, seps[1])
|
|
442
|
+
d = None
|
|
443
|
+
domatch = 1
|
|
444
|
+
ms = re.match(mps['p'], pattern, re.I)
|
|
445
|
+
if ms: # generic pattern matches
|
|
446
|
+
pidx = int(ms.group(1))
|
|
447
|
+
pattern = PgOPT.params['GP'][pidx] if 'GP' in PgOPT.params else None
|
|
448
|
+
if not pattern: PgLOG.pglog("{}: MISS value per option -GP for matching general pattern '{}'".format(fname, replace), PgOPT.PGOPT['extlog'])
|
|
449
|
+
domatch = 1
|
|
450
|
+
|
|
451
|
+
if domatch:
|
|
452
|
+
ms = re.match(mps['c'], pattern, re.I) # current date
|
|
453
|
+
if ms:
|
|
454
|
+
pattern = ms.group(1)
|
|
455
|
+
d = PgOPT.params['CD']
|
|
456
|
+
h = PgOPT.params['CH']
|
|
457
|
+
domatch = 0
|
|
458
|
+
if domatch and (not date or re.match(mps['s'], pattern, re.I)): continue
|
|
459
|
+
|
|
460
|
+
if domatch:
|
|
461
|
+
ms = re.match(mps['m'], pattern, re.I)
|
|
462
|
+
if ms:
|
|
463
|
+
pattern = ms.group(1)
|
|
464
|
+
if intv and len(intv) == 7 and intv[6] and re.search(mps['d'], date):
|
|
465
|
+
ms = re.search(mps['d'], date)
|
|
466
|
+
d = ms.group(1)
|
|
467
|
+
d = (intv[6] - 1) if d >= 28 else int(d*intv/30)
|
|
468
|
+
if pattern == "C":
|
|
469
|
+
pattern = chr(65 + d) # upper case, chr(65) is A
|
|
470
|
+
elif pattern == "c":
|
|
471
|
+
pattern = chr(97 + d) # lower case, chr(97) is a
|
|
472
|
+
else:
|
|
473
|
+
pattern = d + 1 # numeric, start from 1
|
|
474
|
+
d = None
|
|
475
|
+
domatch = 0
|
|
476
|
+
else:
|
|
477
|
+
PgLOG.pglog("{}: MISS month fraction for '{}'".format(fname, replace), PgOPT.PGOPT['emllog'])
|
|
478
|
+
|
|
479
|
+
if domatch:
|
|
480
|
+
ms = re.match(mps['n'], pattern, re.I)
|
|
481
|
+
if ms:
|
|
482
|
+
pattern = ms.group(1)
|
|
483
|
+
if not bdate: (bdate, bhour) = addfrequency(date, hour, intv, 0)
|
|
484
|
+
plen = len(pattern)
|
|
485
|
+
if re.match(r'^D', pattern):
|
|
486
|
+
diff = PgUtil.diffdate(date, bdate)
|
|
487
|
+
else:
|
|
488
|
+
diff = PgUtil.diffdatehour(date, hour, bdate, bhour)
|
|
489
|
+
pattern = "{:0{}}".format(diff, plen)
|
|
490
|
+
domatch = 0
|
|
491
|
+
|
|
492
|
+
if domatch:
|
|
493
|
+
ms = re.match(mps['b'], pattern, re.I)
|
|
494
|
+
if ms:
|
|
495
|
+
pattern = ms.group(1)
|
|
496
|
+
d = date
|
|
497
|
+
elif 'UB' in PgOPT.params:
|
|
498
|
+
d = date
|
|
499
|
+
|
|
500
|
+
if d and intv: # beginning time of update period
|
|
501
|
+
if bdate:
|
|
502
|
+
d = bdate
|
|
503
|
+
h = bhour
|
|
504
|
+
else:
|
|
505
|
+
(d, h) = addfrequency(d, hour, intv, 0)
|
|
506
|
+
else:
|
|
507
|
+
ms = re.match(mps['w'], pattern, re.I)
|
|
508
|
+
if ms: # back to the nearest Wed
|
|
509
|
+
pattern = ms.group(1)
|
|
510
|
+
wd = PgUtil.get_weekday(date)
|
|
511
|
+
if wd < 3:
|
|
512
|
+
wd += 4
|
|
513
|
+
else:
|
|
514
|
+
wd -= 3
|
|
515
|
+
d = PgUtil.adddate(date, 0, 0, -wd) if (wd > 0) else date
|
|
516
|
+
else:
|
|
517
|
+
d = date
|
|
518
|
+
h = hour
|
|
519
|
+
|
|
520
|
+
if d: pattern = PgUtil.format_datehour(d, h, pattern)
|
|
521
|
+
fname = re.sub(replace, pattern, fname, 1)
|
|
522
|
+
|
|
523
|
+
return fname
|
|
524
|
+
|
|
525
|
+
#
|
|
526
|
+
# get next display order of an archived data file of given dataset (and group)
|
|
527
|
+
#
|
|
528
|
+
def get_next_exec_order(dsid, next):
|
|
529
|
+
|
|
530
|
+
global CORDERS
|
|
531
|
+
|
|
532
|
+
if not dsid:
|
|
533
|
+
CORDERS = {} # reinitial lize cached display orders
|
|
534
|
+
return
|
|
535
|
+
|
|
536
|
+
if dsid not in CORDERS:
|
|
537
|
+
if next:
|
|
538
|
+
pgrec = PgDBI.pgget("dlupdt", "max(execorder) max_order", "dsid = '{}'".format(dsid), PgOPT.PGOPT['extlog'])
|
|
539
|
+
CORDERS[dsid] = pgrec['max_order'] if pgrec else 0
|
|
540
|
+
CORDERS[dsid] += 1
|
|
541
|
+
|
|
542
|
+
return CORDERS[dsid]
|
|
543
|
+
|
|
544
|
+
#
|
|
545
|
+
# execute specialist specified command
|
|
546
|
+
#
|
|
547
|
+
def executable_command(cmd, file, dsid, edate, ehour, rfiles = None):
|
|
548
|
+
|
|
549
|
+
if not cmd or re.match(r'^#', cmd): return None
|
|
550
|
+
if re.search(r'\$', cmd): cmd = PgLOG.replace_environments(cmd, None, PgOPT.PGOPT['emlerr'])
|
|
551
|
+
if file:
|
|
552
|
+
ms = re.search(r'__(FN|FNAME|FILENAME)__', cmd)
|
|
553
|
+
if ms:
|
|
554
|
+
cmd = re.sub(r'__{}__'.format(ms.group(1)), file, cmd)
|
|
555
|
+
elif re.search(r'(-LF|-RF|-SF)', cmd):
|
|
556
|
+
ms = re.search(r'(-LF|-RF|-SF)', cmd)
|
|
557
|
+
cmd = re.sub(ms.group(1), file, cmd)
|
|
558
|
+
elif re.search(r'/$', cmd):
|
|
559
|
+
cmd += file
|
|
560
|
+
if re.search(r'(^|\s|\||\S/)msrcp\s', cmd):
|
|
561
|
+
cmd += " file"
|
|
562
|
+
elif re.search(r'(^|\s|\||\S/)(cp|mv)\s', cmd):
|
|
563
|
+
cmd += " ."
|
|
564
|
+
elif cmd.find(file) < 0 and re.search(r'(^|\s|\||\S/)(rm\s|tar\s.+\.tar$)', cmd):
|
|
565
|
+
cmd += " file"
|
|
566
|
+
|
|
567
|
+
if re.search(r'-RF', cmd):
|
|
568
|
+
names = []
|
|
569
|
+
if rfiles:
|
|
570
|
+
for rfile in rfiles:
|
|
571
|
+
if isinstance(rfile, dict):
|
|
572
|
+
names.append(rfile['fname'])
|
|
573
|
+
else:
|
|
574
|
+
names.append(rfile)
|
|
575
|
+
name = ' '.join(names)
|
|
576
|
+
cmd = re.sub(r'-RF', name, cmd, 1)
|
|
577
|
+
if re.search(r'-DS', cmd):
|
|
578
|
+
name = dsid if dsid else ""
|
|
579
|
+
cmd = re.sub(r'-DS', name, cmd, 1)
|
|
580
|
+
if edate and re.search(r'-ED', cmd):
|
|
581
|
+
name = str(edate) if edate else ""
|
|
582
|
+
cmd = re.sub('-ED', name, cmd, 1)
|
|
583
|
+
if re.search(r'-EH', cmd):
|
|
584
|
+
name = str(ehour) if ehour != None else ''
|
|
585
|
+
cmd = re.sub(r'-EH', name, cmd, 1)
|
|
586
|
+
ms = re.search(r'(-SN|-LN)', cmd)
|
|
587
|
+
if ms:
|
|
588
|
+
cmd = re.sub(ms.group(1), PgOPT.params['LN'], cmd, 1)
|
|
589
|
+
if re.search(r'-LI', cmd):
|
|
590
|
+
name = str(PgOPT.PGOPT['lindex']) if PgOPT.PGOPT['lindex'] else ''
|
|
591
|
+
cmd = re.sub(r'-LI', name, cmd, 1)
|
|
592
|
+
|
|
593
|
+
return cmd
|
|
594
|
+
|
|
595
|
+
#
|
|
596
|
+
# get the local file names
|
|
597
|
+
#
|
|
598
|
+
def get_local_names(lfile, tempinfo, edate = None):
|
|
599
|
+
|
|
600
|
+
locfiles = []
|
|
601
|
+
ehour = tempinfo['ehour']
|
|
602
|
+
if not edate: edate = tempinfo['edate']
|
|
603
|
+
if lfile[0] == '!': # executable for build up local file names
|
|
604
|
+
cmd = executable_command(lfile[1:], None, PgOPT.params['DS'], edate, ehour)
|
|
605
|
+
if not cmd: return 0
|
|
606
|
+
buf = PgLOG.pgsystem(cmd, PgOPT.PGOPT['wrnlog'], 21)
|
|
607
|
+
if not buf: return PgLOG.pglog(lfile + ": NO local filename returned", PgOPT.PGOPT['emlerr'])
|
|
608
|
+
locfiles = re.split('::', buf)
|
|
609
|
+
else:
|
|
610
|
+
lfiles = expand_serial_pattern(lfile)
|
|
611
|
+
lcnt = len(lfiles)
|
|
612
|
+
for i in range(lcnt):
|
|
613
|
+
locfiles.append(replace_pattern(lfiles[i], edate, ehour, tempinfo['FQ']))
|
|
614
|
+
|
|
615
|
+
return locfiles if locfiles else None
|
|
616
|
+
|
|
617
|
+
#
|
|
618
|
+
# expend serial pattern
|
|
619
|
+
#
|
|
620
|
+
def expand_serial_pattern(fname):
|
|
621
|
+
|
|
622
|
+
if not fname: return None
|
|
623
|
+
seps = PgOPT.params['PD']
|
|
624
|
+
|
|
625
|
+
ms = re.search(r'{}S(\d[\d:]+\d)S{}'.format(seps[0], seps[1]), fname)
|
|
626
|
+
if not ms: return [fname]
|
|
627
|
+
rep = "{}S{}S{}".format(seps[0], ms.group(1), seps[1])
|
|
628
|
+
mcs = re.split(':', ms.group(1))
|
|
629
|
+
tlen = len(mcs[0])
|
|
630
|
+
idx = [0]*3
|
|
631
|
+
idx[0] = int(mcs[0])
|
|
632
|
+
idx[1] = int(mcs[1])
|
|
633
|
+
idx[2] = int(mcs[2]) if len(mcs) > 2 else 1
|
|
634
|
+
fns = []
|
|
635
|
+
i = idx[0]
|
|
636
|
+
while i <= idx[1]:
|
|
637
|
+
val = "{:0{}}".format(i, tlen)
|
|
638
|
+
fn = re.sub(rep, val, fname, 1)
|
|
639
|
+
fns.append(fn)
|
|
640
|
+
i += idx[2]
|
|
641
|
+
|
|
642
|
+
return fns
|
|
643
|
+
|
|
644
|
+
#
|
|
645
|
+
# get the remote file names
|
|
646
|
+
#
|
|
647
|
+
def get_remote_names(rfile, rmtrec, rmtinfo, tempinfo, edate = None):
|
|
648
|
+
|
|
649
|
+
rmtfiles = []
|
|
650
|
+
if not edate: edate = tempinfo['edate']
|
|
651
|
+
if rfile[0] == '!': # executable for build up remote file names
|
|
652
|
+
cmd = executable_command(rfile[1:], None, PgOPT.params['DS'], edate, tempinfo['ehour'])
|
|
653
|
+
if not cmd: return None
|
|
654
|
+
rfile = PgLOG.pgsystem(cmd, PgOPT.PGOPT['wrnlog'], 21)
|
|
655
|
+
if not rfile: return PgLOG.pglog(rmtinfo + ": NO remote filename returned", PgOPT.PGOPT['emlerr'])
|
|
656
|
+
rmtfiles = re.split('::', rfile)
|
|
657
|
+
else:
|
|
658
|
+
rfiles = expand_serial_pattern(rfile)
|
|
659
|
+
rcnt = len(rfiles)
|
|
660
|
+
for i in range(rcnt):
|
|
661
|
+
rmtfiles.extend(replace_remote_pattern_times(rfiles[i], rmtrec, rmtinfo, tempinfo, edate))
|
|
662
|
+
|
|
663
|
+
return rmtfiles if rmtfiles else None
|
|
664
|
+
|
|
665
|
+
#
|
|
666
|
+
# get and replace pattern dates/hours for remote files
|
|
667
|
+
#
|
|
668
|
+
def replace_remote_pattern_times(rfile, rmtrec, rmtinfo, tempinfo, edate = None):
|
|
669
|
+
|
|
670
|
+
rfiles = []
|
|
671
|
+
if not edate: edate = tempinfo['edate']
|
|
672
|
+
ehour = tempinfo['ehour']
|
|
673
|
+
freq = tempinfo['FQ']
|
|
674
|
+
(bdate, bhour) = addfrequency(edate, ehour, freq, 0)
|
|
675
|
+
funit = tempinfo['QU'] if tempinfo['QU'] else None
|
|
676
|
+
tintv = rmtrec['tinterval'] if rmtrec['tinterval'] else None
|
|
677
|
+
if not tintv:
|
|
678
|
+
if rmtrec['dindex'] and funit:
|
|
679
|
+
if need_time_interval(rfile, freq): return []
|
|
680
|
+
rfiles = [one_remote_filename(rfile, edate, ehour, tempinfo, None, bdate, bhour)]
|
|
681
|
+
return rfiles
|
|
682
|
+
elif not funit:
|
|
683
|
+
PgLOG.pglog("{}: MISS Update Frequency for given time interval '{}'".format(rmtinfo, tintv), PgOPT.PGOPT['emlerr'])
|
|
684
|
+
return []
|
|
685
|
+
|
|
686
|
+
ms = re.match(r'^(\d*)([YMWDH])$', tintv)
|
|
687
|
+
if ms:
|
|
688
|
+
val = int(ms.group(1)) if len(ms.group(1)) > 0 else 1
|
|
689
|
+
unit = ms.group(2)
|
|
690
|
+
if unit == 'W': val *= 7
|
|
691
|
+
else:
|
|
692
|
+
PgLOG.pglog("{}: time interval '{}' NOT in (Y,M,W,D,H)".format(rmtinfo, tintv), PgOPT.PGOPT['emlerr'])
|
|
693
|
+
return []
|
|
694
|
+
|
|
695
|
+
# check if multiple data periods
|
|
696
|
+
i = 0 # not single period
|
|
697
|
+
if unit == 'H':
|
|
698
|
+
if freq[3] and freq[3] <= val: i = 1
|
|
699
|
+
elif unit == 'D' or unit == 'W':
|
|
700
|
+
if freq[3] or freq[2] and freq[2] <= val: i = 1
|
|
701
|
+
elif unit == 'M':
|
|
702
|
+
if freq[3] or freq[2] or freq[1] and freq[1] <= val: i = 1
|
|
703
|
+
elif unit == 'Y':
|
|
704
|
+
if not freq[0] or freq[0] <= val: i = 1
|
|
705
|
+
|
|
706
|
+
if i == 1:
|
|
707
|
+
rfiles = [one_remote_filename(rfile, edate, ehour, tempinfo, None, bdate, bhour)]
|
|
708
|
+
return rfiles
|
|
709
|
+
|
|
710
|
+
date = edate
|
|
711
|
+
hour = ehour
|
|
712
|
+
# set ending date/hour for multiple data periods
|
|
713
|
+
max = replace_pattern(rmtrec['endtime'], date, 0) if rmtrec['endtime'] else 0
|
|
714
|
+
if max:
|
|
715
|
+
ms = re.match(r'^(\d+-\d+-\d+)', max)
|
|
716
|
+
if ms:
|
|
717
|
+
edate = ms.group(1)
|
|
718
|
+
ms = re.search(r':(\d+)', max)
|
|
719
|
+
if ms: ehour = int(ms.group(1))
|
|
720
|
+
max = 0
|
|
721
|
+
else:
|
|
722
|
+
if freq[1] and max.find(':') > -1:
|
|
723
|
+
maxs = re.split(':', max)
|
|
724
|
+
if len(maxs) == 12:
|
|
725
|
+
mn = 1
|
|
726
|
+
ms = re.match(r'^(\d+)-(\d+)', bdate)
|
|
727
|
+
if ms: mn = int(ms.group(2))
|
|
728
|
+
max = int(maxs[mn - 1])
|
|
729
|
+
else: # use the first one
|
|
730
|
+
max = int(maxs[0])
|
|
731
|
+
|
|
732
|
+
if max:
|
|
733
|
+
if unit == 'H':
|
|
734
|
+
(edate, ehour) = PgUtil.adddatehour(bdate, bhour, 0, 0, 0, max)
|
|
735
|
+
elif unit == 'Y':
|
|
736
|
+
edate = PgUtil.adddate(bdate, max, 0, 0)
|
|
737
|
+
elif unit == 'M':
|
|
738
|
+
edate = PgUtil.adddate(bdate, 0, max, 0)
|
|
739
|
+
elif unit == 'W' or unit == 'D':
|
|
740
|
+
edate = PgUtil.adddate(bdate, 0, 0, max)
|
|
741
|
+
|
|
742
|
+
# set beginning date/hour for multiple data periods
|
|
743
|
+
min = replace_pattern(rmtrec['begintime'], date, 0) if rmtrec['begintime'] else 0
|
|
744
|
+
if min:
|
|
745
|
+
ms = re.match(r'^(\d+-\d+-\d+)', min)
|
|
746
|
+
if ms:
|
|
747
|
+
date = ms.group(1)
|
|
748
|
+
ms = re.search(r':(\d+)', min)
|
|
749
|
+
if ms:
|
|
750
|
+
hour = int(ms.group(1))
|
|
751
|
+
else:
|
|
752
|
+
hour = 0
|
|
753
|
+
min = 0
|
|
754
|
+
else:
|
|
755
|
+
date = bdate
|
|
756
|
+
hour = bhour
|
|
757
|
+
if freq[1] and min.find(':') > -1:
|
|
758
|
+
mins = re.split(':', min)
|
|
759
|
+
if len(mins) == 12:
|
|
760
|
+
mn = 1
|
|
761
|
+
ms = re.match(r'^(\d+)-(\d+)', date)
|
|
762
|
+
if ms: mn = int(ms.group(2))
|
|
763
|
+
min = int(mins[mn-1])
|
|
764
|
+
else: # use the first one
|
|
765
|
+
min = int(mins[0])
|
|
766
|
+
else:
|
|
767
|
+
date = bdate
|
|
768
|
+
hour = bhour
|
|
769
|
+
|
|
770
|
+
if min and not isinstance(min, int): min = int(min)
|
|
771
|
+
gotintv = 0
|
|
772
|
+
intv = [0]*4
|
|
773
|
+
if unit == 'Y':
|
|
774
|
+
intv[0] = val
|
|
775
|
+
gotintv += 1
|
|
776
|
+
if min: date = PgUtil.adddate(date, min, 0, 0)
|
|
777
|
+
elif unit == 'M':
|
|
778
|
+
intv[1] = val
|
|
779
|
+
gotintv += 1
|
|
780
|
+
if min:
|
|
781
|
+
date = PgUtil.adddate(date, 0, min, 0)
|
|
782
|
+
else:
|
|
783
|
+
date = PgUtil.enddate(date, 0, 'M')
|
|
784
|
+
elif unit == 'W' or unit == 'D':
|
|
785
|
+
intv[2] = val
|
|
786
|
+
gotintv += 1
|
|
787
|
+
if min: date = PgUtil.adddate(date, 0, 0, min)
|
|
788
|
+
elif unit == 'H':
|
|
789
|
+
intv[3] = val
|
|
790
|
+
gotintv += 1
|
|
791
|
+
if hour is None or not freq[3]:
|
|
792
|
+
ehour = 23
|
|
793
|
+
hour = 0
|
|
794
|
+
if min: (date, hour) = PgUtil.adddatehour(date, hour, 0, 0, 0, min)
|
|
795
|
+
|
|
796
|
+
if not gotintv:
|
|
797
|
+
PgLOG.pglog("{}: error process time internal '{}'".format(rmtinfo, tintv), PgOPT.PGOPT['emlerr'])
|
|
798
|
+
return []
|
|
799
|
+
|
|
800
|
+
rfiles = []
|
|
801
|
+
i = 0
|
|
802
|
+
while PgUtil.diffdatehour(date, hour, edate, ehour) <= 0:
|
|
803
|
+
rfiles.append(one_remote_filename(rfile, date, hour, tempinfo, intv, bdate, bhour))
|
|
804
|
+
(date, hour) = PgUtil.adddatehour(date, hour, intv[0], intv[1], intv[2], intv[3])
|
|
805
|
+
|
|
806
|
+
return rfiles
|
|
807
|
+
|
|
808
|
+
#
|
|
809
|
+
# adjust date by skip Feb. 29 for leap year
|
|
810
|
+
#
|
|
811
|
+
def adjust_leap(date):
|
|
812
|
+
|
|
813
|
+
(syr, smn, sdy) = re.split('-', date)
|
|
814
|
+
if PgUtil.is_leapyear(int(syr)):
|
|
815
|
+
sdy = '28'
|
|
816
|
+
date = "{}-{}-{}", syr, smn, sdy
|
|
817
|
+
|
|
818
|
+
return date
|
|
819
|
+
|
|
820
|
+
#
|
|
821
|
+
# get one hash array for a single remote file name
|
|
822
|
+
#
|
|
823
|
+
def one_remote_filename(fname, date, hour, tempinfo, intv, bdate, bhour):
|
|
824
|
+
|
|
825
|
+
if tempinfo['NX']:
|
|
826
|
+
(udate, uhour) = PgUtil.adddatehour(date, hour, tempinfo['NX'][0], tempinfo['NX'][1], tempinfo['NX'][2], tempinfo['NX'][3])
|
|
827
|
+
else:
|
|
828
|
+
udate = date,
|
|
829
|
+
uhour = hour
|
|
830
|
+
|
|
831
|
+
if 'CP' in PgOPT.params:
|
|
832
|
+
(vdate, vhour) = addfrequency(PgOPT.PGOPT['CURDATE'], PgOPT.PGOPT['CURHOUR'], tempinfo['FQ'], 1)
|
|
833
|
+
else:
|
|
834
|
+
vdate = PgOPT.PGOPT['CURDATE']
|
|
835
|
+
vhour = PgOPT.PGOPT['CURHOUR']
|
|
836
|
+
|
|
837
|
+
rfile = {}
|
|
838
|
+
if intv is None: intv = tempinfo['FQ']
|
|
839
|
+
rfile['fname'] = replace_pattern(fname, date, hour, intv, 0, bdate, bhour)
|
|
840
|
+
if 'FU' in PgOPT.params or PgUtil.diffdatehour(udate, uhour, vdate, vhour) <= 0:
|
|
841
|
+
if tempinfo['VD'] and PgUtil.diffdatehour(date, hour, tempinfo['VD'], tempinfo['VH']) < 0:
|
|
842
|
+
rfile['ready'] = -1
|
|
843
|
+
else:
|
|
844
|
+
rfile['ready'] = 1
|
|
845
|
+
else:
|
|
846
|
+
rfile['ready'] = 0
|
|
847
|
+
|
|
848
|
+
rfile['amiss'] = 1 if (tempinfo['amiss'] == 'Y') else 0
|
|
849
|
+
rfile['date'] = date
|
|
850
|
+
rfile['hour'] = hour
|
|
851
|
+
if hour is None:
|
|
852
|
+
rfile['time'] = "23:59:59"
|
|
853
|
+
else:
|
|
854
|
+
rfile['time'] = "{:02}:00:00".format(hour)
|
|
855
|
+
|
|
856
|
+
if tempinfo['DC']:
|
|
857
|
+
rfile['rcmd'] = replace_pattern(tempinfo['DC'], date, hour, intv, 0, bdate, bhour)
|
|
858
|
+
else:
|
|
859
|
+
rfile['rcmd'] = None
|
|
860
|
+
|
|
861
|
+
return rfile
|
|
862
|
+
|
|
863
|
+
#
|
|
864
|
+
# record the date/hour for missing data
|
|
865
|
+
#
|
|
866
|
+
def set_miss_time(lfile, locrec, tempinfo, rmonly = 0):
|
|
867
|
+
|
|
868
|
+
setmiss = 1
|
|
869
|
+
mdate = mhour = None
|
|
870
|
+
pgrec = {}
|
|
871
|
+
if rmonly:
|
|
872
|
+
if(not locrec['missdate'] or
|
|
873
|
+
PgUtil.diffdatehour(tempinfo['edate'], tempinfo['ehour'], locrec['missdate'], locrec['misshour'])):
|
|
874
|
+
return setmiss # do not remove if miss times not match
|
|
875
|
+
elif PgUtil.diffdatehour(tempinfo['edate'], tempinfo['ehour'], tempinfo['VD'], tempinfo['VH']) >= 0:
|
|
876
|
+
mdate = tempinfo['edate']
|
|
877
|
+
if tempinfo['ehour'] is not None: mhour = tempinfo['ehour']
|
|
878
|
+
setmiss = 0
|
|
879
|
+
|
|
880
|
+
if locrec['missdate']:
|
|
881
|
+
if not mdate:
|
|
882
|
+
pgrec['missdate'] = pgrec['misshour'] = None
|
|
883
|
+
elif (PgUtil.diffdatehour(mdate, mhour, locrec['missdate'], locrec['misshour']) and
|
|
884
|
+
PgUtil.diffdatehour(locrec['missdate'], locrec['misshour'], tempinfo['VD'], tempinfo['VH']) < 0):
|
|
885
|
+
pgrec['missdate'] = mdate
|
|
886
|
+
pgrec['misshour'] = mhour
|
|
887
|
+
elif mdate:
|
|
888
|
+
pgrec['missdate'] = mdate
|
|
889
|
+
pgrec['misshour'] = mhour
|
|
890
|
+
|
|
891
|
+
if not pgrec:
|
|
892
|
+
if locrec['misshour']:
|
|
893
|
+
if mhour is None or mhour != locrec['misshour']:
|
|
894
|
+
pgrec['misshour'] = mhour
|
|
895
|
+
elif mhour is not None:
|
|
896
|
+
pgrec['misshour'] = mhour
|
|
897
|
+
|
|
898
|
+
if pgrec: PgDBI.pgupdt("dlupdt", pgrec, "lindex = {}".format(locrec['lindex']), PgOPT.PGOPT['extlog'])
|
|
899
|
+
return setmiss
|
|
900
|
+
|
|
901
|
+
#
|
|
902
|
+
# reset next data end/update times
|
|
903
|
+
#
|
|
904
|
+
def reset_update_time(locinfo, locrec, tempinfo, arccnt, endonly):
|
|
905
|
+
|
|
906
|
+
gx = 1 if re.search(r'(^|\s)-GX(\s|$)', locrec['options'], re.I) else 0
|
|
907
|
+
date = tempinfo['edate']
|
|
908
|
+
hour = tempinfo['ehour']
|
|
909
|
+
if not gx and ('UT' in PgOPT.params or arccnt > 0):
|
|
910
|
+
pgrec = get_period_record(locrec['gindex'], PgOPT.params['DS'], locinfo)
|
|
911
|
+
if pgrec:
|
|
912
|
+
ehour = None
|
|
913
|
+
if hour != None:
|
|
914
|
+
ms = re.match(r'^(\d+):', str(pgrec['time_end']))
|
|
915
|
+
if ms: ehour = int(ms.group(1))
|
|
916
|
+
diff = PgUtil.diffdatehour(date, hour, pgrec['date_end'], ehour)
|
|
917
|
+
if 'UT' in PgOPT.params or diff > 0:
|
|
918
|
+
sdpcmd = "sdp -d {} -g {} -ed {}".format(PgOPT.params['DS'][2:], pgrec['gindex'], date)
|
|
919
|
+
if hour != None: sdpcmd += " -et {:02}:59:59".format(hour)
|
|
920
|
+
if PgLOG.pgsystem(sdpcmd, PgLOG.MSGLOG, 32):
|
|
921
|
+
einfo = "{}".format(date)
|
|
922
|
+
if hour != None: einfo += ":{:02}".format(hour)
|
|
923
|
+
PgLOG.pglog("{}: data archive period {} to {}".format(locinfo, ("EXTENDED" if diff > 0 else "CHANGED"), einfo), PgOPT.PGOPT['emllog'])
|
|
924
|
+
|
|
925
|
+
if not tempinfo['FQ'] or endonly and arccnt < 1: return
|
|
926
|
+
if PgUtil.diffdatehour(date, hour, PgOPT.params['CD'], PgOPT.params['CH']) <= 0:
|
|
927
|
+
(date, hour) = addfrequency(date, hour, tempinfo['FQ'], 1)
|
|
928
|
+
date = PgUtil.enddate(date, tempinfo['EP'], tempinfo['QU'], tempinfo['FQ'][6])
|
|
929
|
+
|
|
930
|
+
if 'UT' in PgOPT.params or not locrec['enddate'] or PgUtil.diffdatehour(date, hour, locrec['enddate'], locrec['endhour']) > 0:
|
|
931
|
+
record = {'enddate' : date}
|
|
932
|
+
if hour != None:
|
|
933
|
+
record['endhour'] = hour
|
|
934
|
+
einfo = "end data date:hour {}:{:02}".format(date, hour)
|
|
935
|
+
else:
|
|
936
|
+
einfo = "end data date {}".format(date)
|
|
937
|
+
if 'GZ' in PgOPT.params: einfo += "(UTC)"
|
|
938
|
+
if tempinfo['NX']:
|
|
939
|
+
(date, hour) = PgUtil.adddatehour(date, hour, tempinfo['NX'][0], tempinfo['NX'][1], tempinfo['NX'][2], tempinfo['NX'][3])
|
|
940
|
+
|
|
941
|
+
if(locrec['enddate'] and
|
|
942
|
+
PgDBI.pgupdt("dlupdt", record, "lindex = {}".format(locrec['lindex']), PgOPT.PGOPT['extlog'])):
|
|
943
|
+
PgLOG.pglog("{}: {} {} for NEXT update".format(locinfo, ("set" if arccnt > 0 else "SKIP to"), einfo), PgOPT.PGOPT['emllog'])
|
|
944
|
+
if PgOPT.PGOPT['UCNTL']: reset_data_time(tempinfo['QU'], tempinfo['edate'], tempinfo['ehour'], locrec['lindex'])
|
|
945
|
+
else:
|
|
946
|
+
PgLOG.pglog("{}: {} for NEXT update".format(locinfo, einfo), PgOPT.PGOPT['emllog'])
|
|
947
|
+
else:
|
|
948
|
+
if locrec['endhour'] != None:
|
|
949
|
+
einfo = "end data date:hour {}:{:02}".format(locrec['enddate'], locrec['endhour'])
|
|
950
|
+
else:
|
|
951
|
+
einfo = "end data date {}".format(locrec['enddate'])
|
|
952
|
+
|
|
953
|
+
if 'GZ' in PgOPT.params: einfo += "(UTC)"
|
|
954
|
+
PgLOG.pglog("{}: ALREADY set {} for NEXT update".format(locinfo, einfo), PgOPT.PGOPT['emllog'])
|
|
955
|
+
if PgOPT.PGOPT['UCNTL']: reset_data_time(tempinfo['QU'], tempinfo['edate'], tempinfo['ehour'], locrec['lindex'])
|
|
956
|
+
|
|
957
|
+
|
|
958
|
+
#
|
|
959
|
+
# get period record for sub group
|
|
960
|
+
#
|
|
961
|
+
def get_period_record(gindex, dsid, locinfo):
|
|
962
|
+
|
|
963
|
+
pgrec = PgDBI.pgget("dsperiod", "gindex, date_end, time_end, dorder",
|
|
964
|
+
"dsid = '{}' AND gindex = {} ORDER BY dorder".format(dsid, gindex), PgOPT.PGOPT['extlog'])
|
|
965
|
+
if not pgrec and gindex:
|
|
966
|
+
pgrec = PgDBI.pgget("dsgroup", "pindex", "dsid = '{}' AND gindex = {}".format(dsid, gindex), PgOPT.PGOPT['extlog'])
|
|
967
|
+
if pgrec: pgrec = get_period_record(pgrec['pindex'], dsid, locinfo)
|
|
968
|
+
|
|
969
|
+
if pgrec and pgrec['date_end'] and pgrec['date_end'] == "0000-00-00":
|
|
970
|
+
PgLOG.pglog(locinfo + ": dsperiod.date_end set as '0000-00-00' by 'gatherxml'", PgOPT.PGOPT['emlerr'])
|
|
971
|
+
pgrec = None
|
|
972
|
+
|
|
973
|
+
return pgrec
|
|
974
|
+
|
|
975
|
+
#
|
|
976
|
+
# check if need time interval for remote/server file
|
|
977
|
+
#
|
|
978
|
+
def need_time_interval(fname, freq):
|
|
979
|
+
|
|
980
|
+
units = PgUtil.temporal_pattern_units(fname, PgOPT.params['PD'])
|
|
981
|
+
if not units: return 0 # no temporal pattern found in file name
|
|
982
|
+
|
|
983
|
+
funit = punit = None
|
|
984
|
+
if freq[2] > 0:
|
|
985
|
+
if 'H' in units:
|
|
986
|
+
punit = "Hourly"
|
|
987
|
+
funit = "Daily"
|
|
988
|
+
elif freq[1] > 0:
|
|
989
|
+
if 'H' in units:
|
|
990
|
+
punit = "Hourly"
|
|
991
|
+
elif 'D' in units:
|
|
992
|
+
punit = "Daily"
|
|
993
|
+
if punit: funit = "Monthly"
|
|
994
|
+
elif freq[0] > 0:
|
|
995
|
+
if 'H' in units:
|
|
996
|
+
punit = "Hourly"
|
|
997
|
+
elif 'D' in units:
|
|
998
|
+
punit = "Daily"
|
|
999
|
+
elif 'M' in units:
|
|
1000
|
+
punit = "Monthly"
|
|
1001
|
+
if punit: funit = "Yearly"
|
|
1002
|
+
|
|
1003
|
+
if punit:
|
|
1004
|
+
PgLOG.pglog("{}: Remote File Name seems defined at {} Time Interval for {} Update, ".format(fname, punit, funit) +
|
|
1005
|
+
"specify the Time Interval in remote file record to continue", PgOPT.PGOPT['emllog'])
|
|
1006
|
+
return 1
|
|
1007
|
+
else:
|
|
1008
|
+
return 0
|
|
1009
|
+
|
|
1010
|
+
#
|
|
1011
|
+
# check if local file is a growing one
|
|
1012
|
+
#
|
|
1013
|
+
def is_growing_file(fname, freq):
|
|
1014
|
+
|
|
1015
|
+
units = PgUtil.temporal_pattern_units(fname, PgOPT.params['PD'])
|
|
1016
|
+
if not units: return 1 # no temporal pattern found in file name
|
|
1017
|
+
|
|
1018
|
+
if freq[3] > 0:
|
|
1019
|
+
if 'H' in units: return 0
|
|
1020
|
+
elif freq[2] > 0:
|
|
1021
|
+
if 'H' in units or 'D' in units: return 0
|
|
1022
|
+
elif freq[1] > 0:
|
|
1023
|
+
if 'H' in units or 'D' in units or 'M' in units and not freq[6]: return 0
|
|
1024
|
+
elif freq[0] > 0:
|
|
1025
|
+
return 0
|
|
1026
|
+
|
|
1027
|
+
return 1
|
|
1028
|
+
|
|
1029
|
+
#
|
|
1030
|
+
# add update frequency to date/hour
|
|
1031
|
+
# opt = -1 - minus, 0 - begin time, 1 - add (default)
|
|
1032
|
+
#
|
|
1033
|
+
def addfrequency(date, hour, intv, opt = 1):
|
|
1034
|
+
|
|
1035
|
+
if date and not isinstance(date, str): date = str(date)
|
|
1036
|
+
if not intv: return (date, hour)
|
|
1037
|
+
freq = intv.copy()
|
|
1038
|
+
if opt == 0: # get begin time of next period
|
|
1039
|
+
if freq[3]:
|
|
1040
|
+
if freq[3] == 1: return (date, hour)
|
|
1041
|
+
(date, hour) = PgUtil.adddatehour(date, hour, 0, 0, 0, 1) # add one hour
|
|
1042
|
+
else:
|
|
1043
|
+
if freq[2] == 1: return (date, hour)
|
|
1044
|
+
date = PgUtil.adddate(date, 0, 0, 1) # add one day
|
|
1045
|
+
|
|
1046
|
+
if opt < 1: # negative frequency for minus
|
|
1047
|
+
flen = len(freq)
|
|
1048
|
+
for i in range(flen):
|
|
1049
|
+
if freq[i]: freq[i] = -freq[i]
|
|
1050
|
+
|
|
1051
|
+
if freq[6]: # add fraction month
|
|
1052
|
+
date = PgUtil.addmonth(date, freq[1], freq[6])
|
|
1053
|
+
elif hour != None: # add date/hour
|
|
1054
|
+
(date, hour) = PgUtil.adddatehour(date, hour, freq[0], freq[1], freq[2], freq[3])
|
|
1055
|
+
else: # add date only
|
|
1056
|
+
date = PgUtil.adddate(date, freq[0], freq[1], freq[2])
|
|
1057
|
+
|
|
1058
|
+
return (date, hour)
|
|
1059
|
+
|
|
1060
|
+
#
|
|
1061
|
+
# send a cumtomized email if built during specialist's process
|
|
1062
|
+
#
|
|
1063
|
+
def send_updated_email(lindex, locinfo):
|
|
1064
|
+
|
|
1065
|
+
pgrec = PgDBI.pgget("dlupdt", "emnote", "lindex = {}".format(lindex), PgLOG.LOGERR)
|
|
1066
|
+
if not (pgrec and pgrec['emnote']): return # no customized email info to send
|
|
1067
|
+
if not PgDBI.send_customized_email(locinfo, pgrec['emnote'], PgOPT.PGOPT['emllog']): return
|
|
1068
|
+
PgDBI.pgexec("update dlupdt set emnote = null where lindex = {}".format(lindex), PgLOG.LOGERR) # empty email after sent
|
|
1069
|
+
|
|
1070
|
+
#
|
|
1071
|
+
# validate given local indices
|
|
1072
|
+
#
|
|
1073
|
+
def validate_lindices(cact):
|
|
1074
|
+
|
|
1075
|
+
if (PgOPT.OPTS['LI'][2]&8) == 8: return 0 # already validated
|
|
1076
|
+
zcnt = 0
|
|
1077
|
+
lcnt = len(PgOPT.params['LI'])
|
|
1078
|
+
i = 0
|
|
1079
|
+
while i < lcnt:
|
|
1080
|
+
val = PgOPT.params['LI'][i]
|
|
1081
|
+
if val:
|
|
1082
|
+
if isinstance(val, int):
|
|
1083
|
+
PgOPT.params['LI'][i] = val
|
|
1084
|
+
else:
|
|
1085
|
+
if re.match(r'^(!|<|>|<>)$', val): break
|
|
1086
|
+
PgOPT.params['LI'][i] = int(val)
|
|
1087
|
+
else:
|
|
1088
|
+
PgOPT.params['LI'][i] = 0
|
|
1089
|
+
i += 1
|
|
1090
|
+
if i >= lcnt: # normal locfile index given
|
|
1091
|
+
for i in range(lcnt):
|
|
1092
|
+
val = PgOPT.params['LI'][i]
|
|
1093
|
+
if not val:
|
|
1094
|
+
if cact == "SL":
|
|
1095
|
+
if 'NL' not in PgOPT.params: PgOPT.action_error("Mode option -NL to add new local file record")
|
|
1096
|
+
zcnt += 1
|
|
1097
|
+
elif cact == "SR":
|
|
1098
|
+
PgOPT.action_error("Local File Index 0 is not allowed/n" +
|
|
1099
|
+
"Use Action SL with Mode option -NL to add new record")
|
|
1100
|
+
continue
|
|
1101
|
+
|
|
1102
|
+
if i > 0 and val == PgOPT.params['LI'][i-1]: continue
|
|
1103
|
+
pgrec = PgDBI.pgget("dlupdt", "dsid, specialist", "lindex = {}".format(val), PgOPT.PGOPT['extlog'])
|
|
1104
|
+
if not pgrec:
|
|
1105
|
+
PgOPT.action_error("Locfile Index {} is not in RDADB".format(val))
|
|
1106
|
+
elif PgOPT.OPTS[PgOPT.PGOPT['CACT']][2] > 0:
|
|
1107
|
+
if pgrec['specialist'] == PgLOG.PGLOG['CURUID']:
|
|
1108
|
+
PgOPT.params['MD'] = 1
|
|
1109
|
+
else:
|
|
1110
|
+
PgOPT.validate_dsowner("dsupdt", pgrec['dsid'])
|
|
1111
|
+
else: # found none-equal condition sign
|
|
1112
|
+
pgrec = PgDBI.pgmget("dlupdt", "DISTINCT lindex", PgDBI.get_field_condition("lindex", PgOPT.params['LI'], 0, 1), PgOPT.PGOPT['extlog'])
|
|
1113
|
+
if not pgrec: PgOPT.action_error("No update record matches given Locfile Index condition")
|
|
1114
|
+
PgOPT.params['LI'] = pgrec['lindex']
|
|
1115
|
+
|
|
1116
|
+
PgOPT.OPTS['LI'][2] |= 8 # set validated flag
|
|
1117
|
+
|
|
1118
|
+
return zcnt
|
|
1119
|
+
|
|
1120
|
+
#
|
|
1121
|
+
# validate given control indices
|
|
1122
|
+
#
|
|
1123
|
+
def validate_cindices(cact):
|
|
1124
|
+
|
|
1125
|
+
if (PgOPT.OPTS['CI'][2] & 8) == 8: return 0 # already validated
|
|
1126
|
+
zcnt = 0
|
|
1127
|
+
if 'CI' in PgOPT.params:
|
|
1128
|
+
ccnt = len(PgOPT.params['CI'])
|
|
1129
|
+
i = 0
|
|
1130
|
+
while i < ccnt:
|
|
1131
|
+
val = PgOPT.params['CI'][i]
|
|
1132
|
+
if val:
|
|
1133
|
+
if isinstance(val, int):
|
|
1134
|
+
PgOPT.params['CI'][i] = val
|
|
1135
|
+
else:
|
|
1136
|
+
if re.match(r'^(!|<|>|<>)$', val): break
|
|
1137
|
+
PgOPT.params['CI'][i] = int(val)
|
|
1138
|
+
else:
|
|
1139
|
+
PgOPT.params['CI'][i] = 0
|
|
1140
|
+
i += 1
|
|
1141
|
+
if i >= ccnt: # normal locfile index given
|
|
1142
|
+
for i in range(ccnt):
|
|
1143
|
+
val = PgOPT.params['CI'][i]
|
|
1144
|
+
if not val:
|
|
1145
|
+
if cact == 'SC':
|
|
1146
|
+
if 'NC' in PgOPT.params:
|
|
1147
|
+
PgOPT.params['CI'][i] = 0
|
|
1148
|
+
zcnt += 1
|
|
1149
|
+
else:
|
|
1150
|
+
PgOPT.action_error("Mode option -NC to add new update control record")
|
|
1151
|
+
continue
|
|
1152
|
+
|
|
1153
|
+
if i > 0 and val == PgOPT.params['CI'][i-1]: continue
|
|
1154
|
+
pgrec = PgDBI.pgget("dcupdt", "dsid, specialist", "cindex = {}".format(val), PgOPT.PGOPT['extlog'])
|
|
1155
|
+
if not pgrec:
|
|
1156
|
+
PgOPT.action_error("Control Index {} is not in RDADB".format(val))
|
|
1157
|
+
elif PgOPT.OPTS[PgOPT.PGOPT['CACT']][2] > 0:
|
|
1158
|
+
if pgrec['specialist'] == PgLOG.PGLOG['CURUID']:
|
|
1159
|
+
PgOPT.params['MD'] = 1
|
|
1160
|
+
else:
|
|
1161
|
+
PgOPT.validate_dsowner("dsupdt", pgrec['dsid'])
|
|
1162
|
+
else: # found none-equal condition sign
|
|
1163
|
+
pgrec = PgDBI.pgmget("dcupdt", "DISTINCT cindex", PgDBI.get_field_condition("cindex", PgOPT.params['CI'], 0, 1), PgOPT.PGOPT['extlog'])
|
|
1164
|
+
if not pgrec: PgOPT.action_error("No update control record matches given Index condition")
|
|
1165
|
+
PgOPT.params['CI'] = pgrec['cindex']
|
|
1166
|
+
|
|
1167
|
+
if len(PgOPT.params['CI']) > 1 and PgOPT.PGOPT['ACTS']&PgOPT.PGOPT['CNTLACTS']:
|
|
1168
|
+
PgOPT.action_error("Process one Update Control each time")
|
|
1169
|
+
|
|
1170
|
+
elif 'ID' in PgOPT.params:
|
|
1171
|
+
PgOPT.params['CI'] = cid2cindex(cact, PgOPT.params['ID'], zcnt)
|
|
1172
|
+
|
|
1173
|
+
PgOPT.OPTS['CI'][2] |= 8 # set validated flag
|
|
1174
|
+
|
|
1175
|
+
return zcnt
|
|
1176
|
+
|
|
1177
|
+
#
|
|
1178
|
+
# get control index array from given control IDs
|
|
1179
|
+
#
|
|
1180
|
+
def cid2cindex(cact, cntlids, zcnt):
|
|
1181
|
+
|
|
1182
|
+
count = len(cntlids) if cntlids else 0
|
|
1183
|
+
if count == 0: return None
|
|
1184
|
+
i = 0
|
|
1185
|
+
while i < count:
|
|
1186
|
+
val = cntlids[i]
|
|
1187
|
+
if val and (re.match(r'^(!|<|>|<>)$', val) or val.find('%') > -1): break
|
|
1188
|
+
i += 1
|
|
1189
|
+
if i >= count: # normal control id given
|
|
1190
|
+
indices = [0]*count
|
|
1191
|
+
for i in range(count):
|
|
1192
|
+
val = cntlids[i]
|
|
1193
|
+
if not val:
|
|
1194
|
+
continue
|
|
1195
|
+
elif i and (val == cntlids[i-1]):
|
|
1196
|
+
indices[i] = indices[i-1]
|
|
1197
|
+
continue
|
|
1198
|
+
else:
|
|
1199
|
+
pgrec = PgDBI.pgget("dcupdt", "cindex", "cntlid = '{}'".format(val), PgOPT.PGOPT['extlog'])
|
|
1200
|
+
if pgrec: indices[i] = pgrec['cindex']
|
|
1201
|
+
|
|
1202
|
+
if not indices[i]:
|
|
1203
|
+
if cact == "SC":
|
|
1204
|
+
if 'NC' in PgOPT.params:
|
|
1205
|
+
indices[i] = 0
|
|
1206
|
+
zcnt += 1
|
|
1207
|
+
else:
|
|
1208
|
+
PgOPT.action_error("Control ID {} is not in RDADB,\n".format(val) +
|
|
1209
|
+
"Use Mode Option -NC (-NewControl) to add new Control", cact)
|
|
1210
|
+
else:
|
|
1211
|
+
PgOPT.action_error("Control ID '{}' is not in RDADB".format(val), cact)
|
|
1212
|
+
|
|
1213
|
+
return indices
|
|
1214
|
+
else: # found wildcard and/or none-equal condition sign
|
|
1215
|
+
pgrec = PgDBI.pgmget("dcupdt", "DISTINCT cindex", PgDBI.get_field_condition("cntlid", cntlids, 1, 1), PgOPT.PGOPT['extlog'])
|
|
1216
|
+
if not pgrec: PgOPT.action_error("No Control matches given Control ID condition")
|
|
1217
|
+
|
|
1218
|
+
return pgrec['cindex']
|
|
1219
|
+
|
|
1220
|
+
#
|
|
1221
|
+
# check remote file information
|
|
1222
|
+
#
|
|
1223
|
+
def check_server_file(dcmd, opt, cfile):
|
|
1224
|
+
|
|
1225
|
+
sfile = info = type = None
|
|
1226
|
+
PgLOG.PGLOG['SYSERR'] = PgOPT.PGOPT['STATUS'] = ''
|
|
1227
|
+
docheck = 1
|
|
1228
|
+
copt = opt|256
|
|
1229
|
+
|
|
1230
|
+
ms = re.search(r'(^|\s|\||\S/)rdacp\s+(.+)$', dcmd)
|
|
1231
|
+
if ms:
|
|
1232
|
+
buf = ms.group(2)
|
|
1233
|
+
type = "RDACP"
|
|
1234
|
+
docheck = 0
|
|
1235
|
+
ms = re.match(r'^(-\w+)', buf)
|
|
1236
|
+
while ms:
|
|
1237
|
+
flg = ms.group(1)
|
|
1238
|
+
buf = re.sub('^-\w+\s+'.format(flg), '', buf, 1) # remove options
|
|
1239
|
+
if flg != "-r": # no option value
|
|
1240
|
+
m = re.match(r'^(\S+)\s', buf)
|
|
1241
|
+
if not m: break
|
|
1242
|
+
if flg == "-f":
|
|
1243
|
+
sfile = ms.group(1)
|
|
1244
|
+
elif flg == "-fh":
|
|
1245
|
+
target = ms.group(1)
|
|
1246
|
+
buf = re.sub(r'^\S\s+', '', buf, 1) # remove values
|
|
1247
|
+
ms = re.match(r'^(-\w+)', buf)
|
|
1248
|
+
|
|
1249
|
+
if not sfile:
|
|
1250
|
+
ms = re.match(r'^(\S+)', buf)
|
|
1251
|
+
if ms: sfile = ms.group(1)
|
|
1252
|
+
info = PgFile.check_rda_file(sfile, target, copt)
|
|
1253
|
+
|
|
1254
|
+
if docheck:
|
|
1255
|
+
ms = re.search(r'(^|\s|\||\S/)(mv|cp)\s+(.+)$', dcmd)
|
|
1256
|
+
if ms:
|
|
1257
|
+
sfile = ms.group(3)
|
|
1258
|
+
type = "COPY" if ms.group(2) == "cp" else "MOVE"
|
|
1259
|
+
docheck = 0
|
|
1260
|
+
ms = re.match(r'^(-\w+\s+)', sfile)
|
|
1261
|
+
while ms:
|
|
1262
|
+
sfile = re.sub(r'^-\w+\s+', '', sfile, 1) # remove options
|
|
1263
|
+
ms = re.match(r'^(-\w+\s+)', sfile)
|
|
1264
|
+
ms = re.match(r'^(\S+)\s', sfile)
|
|
1265
|
+
if ms: sfile = ms.group(1)
|
|
1266
|
+
info = PgFile.check_local_file(sfile, copt)
|
|
1267
|
+
|
|
1268
|
+
if docheck:
|
|
1269
|
+
ms = re.search(r'(^|\s|\||\S/)tar\s+(-\w+)\s+(\S+\.tar)\s+(\S+)$', dcmd)
|
|
1270
|
+
if ms:
|
|
1271
|
+
sfile = ms.group(4)
|
|
1272
|
+
target = ms.group(3)
|
|
1273
|
+
type = "UNTAR" if ms.group(2).find('x') > -1 else "TAR"
|
|
1274
|
+
docheck = 0
|
|
1275
|
+
info = PgFile.check_tar_file(sfile, target, copt)
|
|
1276
|
+
|
|
1277
|
+
if docheck:
|
|
1278
|
+
ms = re.search(r'(^|\s|\||\S/)ncftpget\s(.*)(ftp://\S+)', dcmd, re.I)
|
|
1279
|
+
if ms:
|
|
1280
|
+
sfile = ms.group(3)
|
|
1281
|
+
buf = ms.group(2)
|
|
1282
|
+
type = "FTP"
|
|
1283
|
+
docheck = 0
|
|
1284
|
+
user = pswd = None
|
|
1285
|
+
if buf:
|
|
1286
|
+
ms = re.search(r'(-u\s+|--user=)(\S+)', buf)
|
|
1287
|
+
if ms: user = ms.group(2)
|
|
1288
|
+
ms = re.search(r'(-p\s+|--password=)(\S+)', buf)
|
|
1289
|
+
if ms: pswd = ms.group(2)
|
|
1290
|
+
info = PgFile.check_ftp_file(sfile, copt, user, pswd)
|
|
1291
|
+
|
|
1292
|
+
if docheck:
|
|
1293
|
+
ms = re.search(r'(^|\s|\||\S/)wget(\s.*)https{0,1}://(\S+)', dcmd, re.I)
|
|
1294
|
+
if ms:
|
|
1295
|
+
obuf = ms.group(2)
|
|
1296
|
+
wbuf = ms.group(3)
|
|
1297
|
+
sfile = op.basename(wbuf)
|
|
1298
|
+
slow_web_access(wbuf)
|
|
1299
|
+
type = "WGET"
|
|
1300
|
+
docheck = 0
|
|
1301
|
+
if not obuf or not re.search(r'\s-N\s', obuf): dcmd = re.sub(r'wget', 'wget -N', dcmd, 1)
|
|
1302
|
+
flg = 0
|
|
1303
|
+
if cfile and sfile != cfile:
|
|
1304
|
+
if PgLOG.pgsystem("cp -p {} {}".format(cfile, sfile), PgOPT.PGOPT['emerol'], 4): flg = 1
|
|
1305
|
+
buf = PgLOG.pgsystem(dcmd, PgOPT.PGOPT['wrnlog'], 16+32)
|
|
1306
|
+
info = PgFile.check_local_file(sfile, opt, PgOPT.PGOPT['wrnlog'])
|
|
1307
|
+
if buf:
|
|
1308
|
+
if not info: PgOPT.PGOPT['STATUS'] = buf
|
|
1309
|
+
if re.search(r'Saving to:\s', buf):
|
|
1310
|
+
flg = 0
|
|
1311
|
+
elif not re.search(r'(Server file no newer|not modified on server)', buf):
|
|
1312
|
+
if info: info['note'] = "{}:\n{}".format(dcmd, buf)
|
|
1313
|
+
else:
|
|
1314
|
+
if info: info['note'] = dcmd + ": Failed checking new file"
|
|
1315
|
+
if flg: PgLOG.pgsystem("rm -rf " + sfile, PgOPT.PGOPT['emerol'], 4)
|
|
1316
|
+
|
|
1317
|
+
if docheck:
|
|
1318
|
+
ms = re.match(r'^(\S+)\s+(.+)$', dcmd)
|
|
1319
|
+
if ms:
|
|
1320
|
+
buf = ms.group(2)
|
|
1321
|
+
type = op.basename(ms.group(1)).upper()
|
|
1322
|
+
files = re.split(' ', buf)
|
|
1323
|
+
for file in files:
|
|
1324
|
+
if re.match(r'^-\w+', file) or not op.exists(file) or cfile and file == cfile: continue
|
|
1325
|
+
info = PgFile.check_local_file(file, copt)
|
|
1326
|
+
if info:
|
|
1327
|
+
info['data_size'] = 0
|
|
1328
|
+
break
|
|
1329
|
+
sfile = file
|
|
1330
|
+
|
|
1331
|
+
if info:
|
|
1332
|
+
info['ftype'] = type
|
|
1333
|
+
else:
|
|
1334
|
+
if not PgOPT.PGOPT['STATUS']: PgOPT.PGOPT['STATUS'] = PgLOG.PGLOG['SYSERR']
|
|
1335
|
+
if not sfile: PgLOG.pglog(dcmd + ": NO enough information in command to check file info", PgOPT.PGOPT['errlog'])
|
|
1336
|
+
|
|
1337
|
+
return info
|
|
1338
|
+
|
|
1339
|
+
#
|
|
1340
|
+
# check and sleep if given web site need to be slowdown for accessing
|
|
1341
|
+
#
|
|
1342
|
+
def slow_web_access(wbuf):
|
|
1343
|
+
|
|
1344
|
+
for wsite in WSLOWS:
|
|
1345
|
+
if wbuf.find(wsite) > -1:
|
|
1346
|
+
time.sleep(WSLOWS[wsite])
|
|
1347
|
+
|
|
1348
|
+
#
|
|
1349
|
+
# check remote server/file status information
|
|
1350
|
+
#
|
|
1351
|
+
# return 1 if exists; 0 missed, -1 with error, -2 comand not surported yet
|
|
1352
|
+
# an error message is stored in PgOPT.PGOPT['STATUS'] if not success
|
|
1353
|
+
#
|
|
1354
|
+
def check_server_status(dcmd):
|
|
1355
|
+
|
|
1356
|
+
PgOPT.PGOPT['STATUS'] = ''
|
|
1357
|
+
target = None
|
|
1358
|
+
ms = re.search(r'(^|\s|\||\S/)rdacp\s+(.+)$', dcmd)
|
|
1359
|
+
if ms:
|
|
1360
|
+
buf = ms.group(2)
|
|
1361
|
+
ms = re.search(r'-fh\s+(\S+)', buf)
|
|
1362
|
+
if ms: target = ms.group(1)
|
|
1363
|
+
ms = re.search(r'-f\s+(\S+)', buf)
|
|
1364
|
+
if ms:
|
|
1365
|
+
fname = ms.group(1)
|
|
1366
|
+
else:
|
|
1367
|
+
ms = re.match(r'^(-\w+)', buf)
|
|
1368
|
+
while ms:
|
|
1369
|
+
flg = ms.group(1)
|
|
1370
|
+
buf = re.sub(r'^-\w+\s+', '', buf, 1) # remove options
|
|
1371
|
+
if flg != "-r": # no option value
|
|
1372
|
+
if not re.match(r'^\S+\s', buf): break
|
|
1373
|
+
buf = re.sub(r'^\S+\s+', '', buf, 1) # remove values
|
|
1374
|
+
ms = re.match(r'^(-\w+)', buf)
|
|
1375
|
+
ms = re.match(r'^(\S+)', buf)
|
|
1376
|
+
if ms: fname = ms.group(1)
|
|
1377
|
+
if not fname:
|
|
1378
|
+
PgOPT.PGOPT['STATUS'] = dcmd + ": MISS from-file per option -f"
|
|
1379
|
+
return -1
|
|
1380
|
+
if not target:
|
|
1381
|
+
return check_local_status(fname)
|
|
1382
|
+
else:
|
|
1383
|
+
return check_remote_status(target, fname)
|
|
1384
|
+
|
|
1385
|
+
ms = re.search(r'(^|\s|\||\S/)(mv|cp|tar|cnvgrib|grabbufr|pb2nc)\s+(.+)$', dcmd)
|
|
1386
|
+
if ms:
|
|
1387
|
+
buf = ms.group(2)
|
|
1388
|
+
fname = ms.group(3)
|
|
1389
|
+
ms = re.match(r'^(-\w+\s+)', fname)
|
|
1390
|
+
while ms:
|
|
1391
|
+
fname = re.sub(r'^-\w+\s+', '', fname, 1) # remove options
|
|
1392
|
+
ms = re.match(r'^(-\w+\s+)', fname)
|
|
1393
|
+
|
|
1394
|
+
ms = re.match(r'^(\S+)\s+(\S*)', fname)
|
|
1395
|
+
if ms:
|
|
1396
|
+
fname = ms.group(1)
|
|
1397
|
+
if buf == 'tar': target = ms.group(2)
|
|
1398
|
+
|
|
1399
|
+
if target:
|
|
1400
|
+
return check_tar_status(fname, target)
|
|
1401
|
+
else:
|
|
1402
|
+
return check_local_status(fname)
|
|
1403
|
+
|
|
1404
|
+
ms = re.search(r'(^|\s|\||\S/)ncftpget\s(.*)(ftp://[^/]+)(/\S+)', dcmd, re.I)
|
|
1405
|
+
if ms:
|
|
1406
|
+
buf = ms.group(2)
|
|
1407
|
+
target = ms.group(3)
|
|
1408
|
+
fname = ms.group(4)
|
|
1409
|
+
user = pswd = None
|
|
1410
|
+
if buf:
|
|
1411
|
+
ms = re.search(r'(-u\s+|--user=)(\S+)', buf)
|
|
1412
|
+
if ms: user = ms.group(2)
|
|
1413
|
+
ms = re.search(r'(-p\s+|--password=)(\S+)', buf)
|
|
1414
|
+
if ms: pswd = ms.group(2)
|
|
1415
|
+
return check_ftp_status(target, fname, user, pswd)
|
|
1416
|
+
|
|
1417
|
+
ms = re.search(r'(^|\s|\||\S/)wget\s(.*)(https{0,1}://[^/]+)(/\S+)', dcmd, re.I)
|
|
1418
|
+
if ms:
|
|
1419
|
+
buf = ms.group(2)
|
|
1420
|
+
target = ms.group(3)
|
|
1421
|
+
fname = ms.group(4)
|
|
1422
|
+
user = pswd = None
|
|
1423
|
+
if buf:
|
|
1424
|
+
ms = re.search(r'(-u\s+|--user=|--http-user=)(\S+)', buf)
|
|
1425
|
+
if ms: user = ms.group(2)
|
|
1426
|
+
ms = re.search(r'(-p\s+|--password=|--http-passwd=)(\S+)', buf)
|
|
1427
|
+
if ms: pswd = ms.group(2)
|
|
1428
|
+
return check_wget_status(target, fname, user, pswd)
|
|
1429
|
+
|
|
1430
|
+
ms = re.match(r'^\s*(\S+)', dcmd)
|
|
1431
|
+
if ms and PgLOG.valid_command(ms.group(1)):
|
|
1432
|
+
return 0
|
|
1433
|
+
else:
|
|
1434
|
+
PgOPT.PGOPT['STATUS'] = dcmd + ": Invalid command"
|
|
1435
|
+
|
|
1436
|
+
return -2
|
|
1437
|
+
|
|
1438
|
+
#
|
|
1439
|
+
# check status for remote server/file via wget
|
|
1440
|
+
# return PgLOG.SUCCESS if file exist and PgLOG.FAILURE otherwise.
|
|
1441
|
+
# file status message is returned via reference string of $status
|
|
1442
|
+
#
|
|
1443
|
+
def check_wget_status(server, fname, user, pswd):
|
|
1444
|
+
|
|
1445
|
+
cmd = "wget --spider --no-check-certificate "
|
|
1446
|
+
if user or pswd:
|
|
1447
|
+
PgOPT.PGOPT['STATUS'] = "{}{}: {}".format(server, fname, PgLOG.PGLOG['MISSFILE'])
|
|
1448
|
+
return -1
|
|
1449
|
+
|
|
1450
|
+
if user: cmd += "--user={} ".format(user)
|
|
1451
|
+
if pswd: cmd += "--password={} ".format(pswd)
|
|
1452
|
+
cmd += server
|
|
1453
|
+
pname = None
|
|
1454
|
+
i = 0
|
|
1455
|
+
while True:
|
|
1456
|
+
msg = PgLOG.pgsystem(cmd + fname, PgLOG.LOGWRN, 48) # 16+32
|
|
1457
|
+
if msg:
|
|
1458
|
+
if msg.find('Remote file exists') > -1:
|
|
1459
|
+
if pname:
|
|
1460
|
+
PgOPT.PGOPT['STATUS'] = "{}{}: {}".format(server, pname, PgLOG.PGLOG['MISSFILE'])
|
|
1461
|
+
return (-1 if i > PgOPT.PGOPT['PCNT'] else 0)
|
|
1462
|
+
else:
|
|
1463
|
+
return 1
|
|
1464
|
+
elif msg.find('unable to resolve host address') > -1:
|
|
1465
|
+
PgOPT.PGOPT['STATUS'] = server + ": Server Un-accessible"
|
|
1466
|
+
return -2
|
|
1467
|
+
elif msg.find('Remote file does not exist') < 0:
|
|
1468
|
+
PgOPT.PGOPT['STATUS'] = "{}{}: Error check status:\n{}".format(cmd, fname, msg)
|
|
1469
|
+
return -2
|
|
1470
|
+
pname = fname
|
|
1471
|
+
fname = op.dirname(pname)
|
|
1472
|
+
if not fname or fname == "/":
|
|
1473
|
+
PgOPT.PGOPT['STATUS'] = "{}{}: {}".format(server, pname, PgLOG.PGLOG['MISSFILE'])
|
|
1474
|
+
return -1
|
|
1475
|
+
fname += "/"
|
|
1476
|
+
i += 1
|
|
1477
|
+
|
|
1478
|
+
#
|
|
1479
|
+
# check status for remote server/file via check_ftp_file()
|
|
1480
|
+
# return PgLOG.SUCCESS if file exist and PgLOG.FAILURE otherwise.
|
|
1481
|
+
# file status message is returned via reference string of $status
|
|
1482
|
+
#
|
|
1483
|
+
def check_ftp_status(server, fname, user, pswd):
|
|
1484
|
+
|
|
1485
|
+
cmd = "ncftpls "
|
|
1486
|
+
if user: cmd += "-u {} ".format(user)
|
|
1487
|
+
if pswd: cmd += "-p {} ".format(pswd)
|
|
1488
|
+
cmd += server
|
|
1489
|
+
pname = None
|
|
1490
|
+
i = 0
|
|
1491
|
+
while True:
|
|
1492
|
+
msg = PgLOG.pgsystem(cmd + fname, PgLOG.LOGWRN, 272) # 16+256
|
|
1493
|
+
if PgLOG.PGLOG['SYSERR']:
|
|
1494
|
+
if PgLOG.PGLOG['SYSERR'].find('unknown host') > -1:
|
|
1495
|
+
PgOPT.PGOPT['STATUS'] = server + ": Server Un-accessible"
|
|
1496
|
+
return -2
|
|
1497
|
+
elif PgLOG.PGLOG['SYSERR'].find('Failed to change directory') < 0:
|
|
1498
|
+
PgOPT.PGOPT['STATUS'] = "{}{}: Error check status:\n{}".format(server, fname, PgLOG.PGLOG['SYSERR'])
|
|
1499
|
+
return -2
|
|
1500
|
+
elif not msg:
|
|
1501
|
+
PgOPT.PGOPT['STATUS'] = "{}{}: {}".format(server, fname, PgLOG.PGLOG['MISSFILE'])
|
|
1502
|
+
return -1 if i >= PgOPT.PGOPT['PCNT'] else 0
|
|
1503
|
+
elif pname:
|
|
1504
|
+
PgOPT.PGOPT['STATUS'] = "{}{}: {}".format(server, pname, PgLOG.PGLOG['MISSFILE'])
|
|
1505
|
+
return -1 if i > PgOPT.PGOPT['PCNT'] else 0
|
|
1506
|
+
else:
|
|
1507
|
+
return 1
|
|
1508
|
+
|
|
1509
|
+
pname = fname
|
|
1510
|
+
fname = op.dirname(pname)
|
|
1511
|
+
if not fname or fname == "/":
|
|
1512
|
+
PgOPT.PGOPT['STATUS'] = "{}{}: {}".format(server, pname, PgLOG.PGLOG['MISSFILE'])
|
|
1513
|
+
return -1
|
|
1514
|
+
i += 1
|
|
1515
|
+
|
|
1516
|
+
#
|
|
1517
|
+
# check remote server status
|
|
1518
|
+
#
|
|
1519
|
+
def check_remote_status(host, fname):
|
|
1520
|
+
|
|
1521
|
+
pname = None
|
|
1522
|
+
i = 0
|
|
1523
|
+
while True:
|
|
1524
|
+
msg = PgLOG.pgsystem("{}-sync {}".format(host, fname), PgLOG.LOGWRN, 272) # 16+256
|
|
1525
|
+
if msg:
|
|
1526
|
+
for line in re.split('\n', msg):
|
|
1527
|
+
info = PgFile.remote_file_stat(line, 0)
|
|
1528
|
+
if info:
|
|
1529
|
+
if pname:
|
|
1530
|
+
PgOPT.PGOPT['STATUS'] = "{}-{}: {}".format(host, pname. PgLOG.PGLOG['MISSFILE'])
|
|
1531
|
+
return -1 if i > PgOPT.PGOPT['PCNT'] else 0
|
|
1532
|
+
else:
|
|
1533
|
+
return 1
|
|
1534
|
+
if PgLOG.PGLOG['SYSERR'] and PgLOG.PGLOG['SYSERR'].find(PgLOG.PGLOG['MISSFILE']) < 0:
|
|
1535
|
+
PgOPT.PGOPT['STATUS'] = "{}-sync {}: Error check status:\n{}".format(host, fname, PgLOG.PGLOG['SYSERR'])
|
|
1536
|
+
return -2
|
|
1537
|
+
|
|
1538
|
+
pname = fname
|
|
1539
|
+
fname = op.dirname(pname)
|
|
1540
|
+
if not fname or fname == "/":
|
|
1541
|
+
PgOPT.PGOPT['STATUS'] = "{}-{}: {}".format(host, pname, PgLOG.PGLOG['MISSFILE'])
|
|
1542
|
+
return -1
|
|
1543
|
+
i += 1
|
|
1544
|
+
|
|
1545
|
+
#
|
|
1546
|
+
# check local disk status
|
|
1547
|
+
#
|
|
1548
|
+
def check_local_status(fname):
|
|
1549
|
+
|
|
1550
|
+
pname = None
|
|
1551
|
+
i = 0
|
|
1552
|
+
while True:
|
|
1553
|
+
if op.exists(fname):
|
|
1554
|
+
if pname:
|
|
1555
|
+
PgOPT.PGOPT['STATUS'] = "{}: {}".format(pname, PgLOG.PGLOG['MISSFILE'])
|
|
1556
|
+
return -1 if i > PgOPT.PGOPT['PCNT'] else 0
|
|
1557
|
+
else:
|
|
1558
|
+
return 1
|
|
1559
|
+
if PgLOG.PGLOG['SYSERR'] and PgLOG.PGLOG['SYSERR'].find(PgLOG.PGLOG['MISSFILE']) < 0:
|
|
1560
|
+
PgOPT.PGOPT['STATUS'] = "{}: Error check status:\n{}".format(fname, PgLOG.PGLOG['SYSERR'])
|
|
1561
|
+
return -2
|
|
1562
|
+
|
|
1563
|
+
pname = fname
|
|
1564
|
+
fname = op.dirname(pname)
|
|
1565
|
+
if not fname or fname == "/":
|
|
1566
|
+
PgOPT.PGOPT['STATUS'] = "{}: {}".format(pname, PgLOG.PGLOG['MISSFILE'])
|
|
1567
|
+
return -1
|
|
1568
|
+
i += 1
|
|
1569
|
+
|
|
1570
|
+
#
|
|
1571
|
+
# check tar file status
|
|
1572
|
+
#
|
|
1573
|
+
def check_tar_status(fname, target):
|
|
1574
|
+
|
|
1575
|
+
stat = check_local_status(fname)
|
|
1576
|
+
if stat < 1: return stat
|
|
1577
|
+
msg = PgLOG.pgsystem("tar -tvf {} {}".format(fname, target), PgLOG.LOGWRN, 272) # 16+256
|
|
1578
|
+
if msg:
|
|
1579
|
+
for line in re.split('\n', msg):
|
|
1580
|
+
if PgFile.tar_file_stat(line, 0): return 1
|
|
1581
|
+
|
|
1582
|
+
if not PgLOG.PGLOG['SYSERR'] or PgLOG.PGLOG['SYSERR'].find('Not found in archive') > -1:
|
|
1583
|
+
PgOPT.PGOPT['STATUS'] = "{}: Not found in tar file {}".format(target, fname)
|
|
1584
|
+
return 0
|
|
1585
|
+
else:
|
|
1586
|
+
PgOPT.PGOPT['STATUS'] = "{}: Error check tar file {}:\n{}".format(target, fname, PgLOG.PGLOG['SYSERR'])
|
|
1587
|
+
return -1
|
|
1588
|
+
|
|
1589
|
+
#
|
|
1590
|
+
# count directories with temoral patterns in given path
|
|
1591
|
+
#
|
|
1592
|
+
def count_pattern_path(dcmd):
|
|
1593
|
+
|
|
1594
|
+
getpath = 1
|
|
1595
|
+
ms = re.search(r'(^|\s|\||\S/)rdacp\s+(.+)$', dcmd)
|
|
1596
|
+
if ms:
|
|
1597
|
+
path = ms.group(2)
|
|
1598
|
+
getpath = 0
|
|
1599
|
+
ms = re.search(r'-f\s+(\S+)', path)
|
|
1600
|
+
if ms:
|
|
1601
|
+
path = ms.group(1)
|
|
1602
|
+
else:
|
|
1603
|
+
ms = re.match(r'^(-\w+)', path)
|
|
1604
|
+
while ms:
|
|
1605
|
+
flg = ms.group(1)
|
|
1606
|
+
path = re.sub(r'^-\w+\s+', '', path, 1) # remove options
|
|
1607
|
+
if flg != "-r": # no option value
|
|
1608
|
+
ms = re.match(r'^(\S+)\s', path)
|
|
1609
|
+
if not ms: break
|
|
1610
|
+
path = re.sub(r'^\S+\s+', '', path, 1) # remove values
|
|
1611
|
+
ms = re.match(r'^(-\w+)', path)
|
|
1612
|
+
ms = re.match(r'^(\S+)', path)
|
|
1613
|
+
if ms: path = ms.group(1)
|
|
1614
|
+
if not path: return PgLOG.pglog(dcmd + ": MISS from-file per option -f", PgOPT.PGOPT['emlerr'])
|
|
1615
|
+
|
|
1616
|
+
if getpath:
|
|
1617
|
+
ms = re.search(r'(^|\s|\||\S/)(mv|cp|tar|cnvgrib|grabbufr|pb2nc)\s+(.+)$', dcmd)
|
|
1618
|
+
if ms:
|
|
1619
|
+
path = ms.group(3)
|
|
1620
|
+
getpath = 0
|
|
1621
|
+
ms = re.match(r'^-\w+\s', path)
|
|
1622
|
+
while ms:
|
|
1623
|
+
path = re.sub(r'^-\w+\s+', '', path, 1) # remove options
|
|
1624
|
+
ms = re.match(r'^-\w+\s', path)
|
|
1625
|
+
ms = re.match(r'^(\S+)\s+(\S*)', path)
|
|
1626
|
+
if ms: path = ms.group(1)
|
|
1627
|
+
|
|
1628
|
+
if getpath:
|
|
1629
|
+
ms = re.search(r'(^|\s|\||\S/)(ncftpget|wget)\s(.*)(ftp|http|https)://[^/]+(/\S+)', dcmd, re.I)
|
|
1630
|
+
if ms: path = ms.group(5)
|
|
1631
|
+
|
|
1632
|
+
if not path: return PgLOG.pglog(dcmd + ": Unkown command to count pattern path", PgOPT.PGOPT['emlerr'])
|
|
1633
|
+
pcnt = path.find(PgOPT.params['PD'][0])
|
|
1634
|
+
if pcnt > 0:
|
|
1635
|
+
path = path[pcnt:]
|
|
1636
|
+
p = re.findall(r'/', path)
|
|
1637
|
+
pcnt = len(p) + 1
|
|
1638
|
+
else:
|
|
1639
|
+
pcnt = 1
|
|
1640
|
+
|
|
1641
|
+
return pcnt
|
|
1642
|
+
|
|
1643
|
+
#
|
|
1644
|
+
# check error message for download action
|
|
1645
|
+
#
|
|
1646
|
+
def parse_download_error(err, act, sinfo = None):
|
|
1647
|
+
|
|
1648
|
+
derr = ''
|
|
1649
|
+
stat = 0
|
|
1650
|
+
if sinfo:
|
|
1651
|
+
if sinfo['data_size'] == 0:
|
|
1652
|
+
derr = ", empty file"
|
|
1653
|
+
if err: derr += ' ' + err
|
|
1654
|
+
elif sinfo['data_size'] < PgLOG.PGLOG['MINSIZE']:
|
|
1655
|
+
derr = ", small file({}B)".format(sinfo['data_size'])
|
|
1656
|
+
if err: derr += ' ' + err
|
|
1657
|
+
else:
|
|
1658
|
+
stat = 1
|
|
1659
|
+
elif err:
|
|
1660
|
+
derr = err
|
|
1661
|
+
if (err.find('command not found') > -1 or
|
|
1662
|
+
err.find('403 Forbidden') > -1):
|
|
1663
|
+
stat = -2
|
|
1664
|
+
elif (act == "wget" and err.find('404 Not Found') > -1 or
|
|
1665
|
+
act == "UNTAR" and err.find('Not found in archive') > -1 or
|
|
1666
|
+
act == "ncftpget" and err.find('Failed to open file') > -1 or
|
|
1667
|
+
err.find(PgLOG.PGLOG['MISSFILE']) > -1):
|
|
1668
|
+
derr = PgLOG.PGLOG['MISSFILE']
|
|
1669
|
+
else:
|
|
1670
|
+
stat = -1
|
|
1671
|
+
|
|
1672
|
+
return (stat, derr)
|
|
1673
|
+
|
|
1674
|
+
#
|
|
1675
|
+
# cache update control information
|
|
1676
|
+
#
|
|
1677
|
+
def cache_update_control(cidx, dolock = 0):
|
|
1678
|
+
|
|
1679
|
+
cstr = "C{}".format(cidx)
|
|
1680
|
+
pgrec = PgDBI.pgget("dcupdt", "*", "cindex = {}".format(cidx), PgOPT.PGOPT['emlerr'])
|
|
1681
|
+
if not pgrec: return PgLOG.pglog(cstr + ": update control record NOT in RDADB", PgOPT.PGOPT['errlog'])
|
|
1682
|
+
if pgrec['dsid']:
|
|
1683
|
+
if 'DS' not in PgOPT.params: PgOPT.params['DS'] = pgrec['dsid']
|
|
1684
|
+
cstr = "{}-{}".format(PgOPT.params['DS'], cstr)
|
|
1685
|
+
if PgOPT.params['DS'] != pgrec['dsid']:
|
|
1686
|
+
return PgLOG.pglog("{}: Control dataset {} NOT match".format(cstr, pgrec['dsid']), PgOPT.PGOPT['emlerr'])
|
|
1687
|
+
|
|
1688
|
+
if pgrec['hostname'] and not valid_control_host(cstr, pgrec['hostname'], PgOPT.PGOPT['emlerr']): return PgLOG.FAILURE
|
|
1689
|
+
if not ('ED' in PgOPT.params or PgOPT.valid_data_time(pgrec, cstr, PgOPT.PGOPT['emlerr'])): return PgLOG.FAILURE
|
|
1690
|
+
if dolock and PgLock.lock_update_control(cidx, 1, PgOPT.PGOPT['emlerr']) <= 0: return PgLOG.FAILURE
|
|
1691
|
+
if PgLOG.PGLOG['DSCHECK']: PgCMD.set_dscheck_attribute("oindex", cidx)
|
|
1692
|
+
if pgrec['updtcntl']:
|
|
1693
|
+
if pgrec['updtcntl'].find('A') > -1: PgOPT.params['CA'] = 1
|
|
1694
|
+
if pgrec['updtcntl'].find('B') > -1: PgOPT.params['UB'] = 1
|
|
1695
|
+
if pgrec['updtcntl'].find('C') > -1: PgOPT.params['CP'] = 1
|
|
1696
|
+
if pgrec['updtcntl'].find('E') > -1: PgOPT.params['RE'] = 1
|
|
1697
|
+
if pgrec['updtcntl'].find('F') > -1: PgOPT.params['FU'] = 1
|
|
1698
|
+
if pgrec['updtcntl'].find('G') > -1:
|
|
1699
|
+
PgOPT.params['GZ'] = 1
|
|
1700
|
+
PgLOG.PGLOG['GMTZ'] = PgUtil.diffgmthour()
|
|
1701
|
+
|
|
1702
|
+
if pgrec['updtcntl'].find('M') > -1: PgOPT.params['MU'] = 1
|
|
1703
|
+
if pgrec['updtcntl'].find('N') > -1: PgOPT.params['CN'] = 1
|
|
1704
|
+
if pgrec['updtcntl'].find('O') > -1: PgOPT.params['MO'] = 1
|
|
1705
|
+
if pgrec['updtcntl'].find('Y') > -1: PgLOG.PGLOG['NOLEAP'] = PgOPT.params['NY'] = 1
|
|
1706
|
+
if pgrec['updtcntl'].find('Z') > -1 and 'VS' not in PgOPT.params:
|
|
1707
|
+
PgLOG.PGLOG['MINSIZE'] = PgOPT.params['VS'] = 0
|
|
1708
|
+
|
|
1709
|
+
if pgrec['emailcntl'] != 'A':
|
|
1710
|
+
if pgrec['emailcntl'] == "N":
|
|
1711
|
+
PgOPT.params['NE'] = 1
|
|
1712
|
+
PgLOG.PGLOG['LOGMASK'] &= ~PgLOG.EMLALL # turn off all email acts
|
|
1713
|
+
elif pgrec['emailcntl'] == "S":
|
|
1714
|
+
PgOPT.params['SE'] = 1
|
|
1715
|
+
PgOPT.PGOPT['emllog'] |= PgLOG.EMEROL
|
|
1716
|
+
elif pgrec['emailcntl'] == "E":
|
|
1717
|
+
PgOPT.params['EE'] = 1
|
|
1718
|
+
elif pgrec['emailcntl'] == "B":
|
|
1719
|
+
PgOPT.params['SE'] = 1
|
|
1720
|
+
PgOPT.params['EE'] = 1
|
|
1721
|
+
PgOPT.PGOPT['emllog'] |= PgLOG.EMEROL
|
|
1722
|
+
|
|
1723
|
+
if pgrec['errorcntl'] != 'N':
|
|
1724
|
+
if pgrec['errorcntl'] == "I":
|
|
1725
|
+
PgOPT.params['IE'] = 1
|
|
1726
|
+
elif pgrec['errorcntl'] == "Q":
|
|
1727
|
+
PgOPT.params['QE'] = 1
|
|
1728
|
+
|
|
1729
|
+
if pgrec['keepfile'] != 'N':
|
|
1730
|
+
if pgrec['keepfile'] == "S":
|
|
1731
|
+
PgOPT.params['KS'] = 1
|
|
1732
|
+
elif pgrec['keepfile'] == "R":
|
|
1733
|
+
PgOPT.params['KR'] = 1
|
|
1734
|
+
elif pgrec['keepfile'] == "B":
|
|
1735
|
+
PgOPT.params['KR'] = 1
|
|
1736
|
+
PgOPT.params['KS'] = 1
|
|
1737
|
+
|
|
1738
|
+
if pgrec['houroffset'] and 'HO' not in PgOPT.params: PgOPT.params['HO'] = [pgrec['houroffset']]
|
|
1739
|
+
if pgrec['emails'] and 'CC' not in PgOPT.params: PgLOG.add_carbon_copy(pgrec['emails'], 1)
|
|
1740
|
+
cache_data_time(cidx)
|
|
1741
|
+
PgOPT.PGOPT['UCNTL'] = pgrec
|
|
1742
|
+
|
|
1743
|
+
return PgLOG.SUCCESS
|
|
1744
|
+
|
|
1745
|
+
#
|
|
1746
|
+
# cache date time info
|
|
1747
|
+
#
|
|
1748
|
+
def cache_data_time(cidx):
|
|
1749
|
+
|
|
1750
|
+
pgrecs = PgDBI.pgmget("dlupdt", "lindex, enddate, endhour", "cindex = {}".format(cidx), PgOPT.PGOPT['emlerr'])
|
|
1751
|
+
cnt = len(pgrecs['lindex']) if pgrecs else 0
|
|
1752
|
+
for i in range(cnt):
|
|
1753
|
+
if not pgrecs['enddate'][i]: continue
|
|
1754
|
+
dhour = pgrecs['endhour'][i] if (pgrecs['endhour'][i] is not None) else 23
|
|
1755
|
+
PgOPT.PGOPT['DTIMES'][pgrecs['lindex'][i]] = "{} {:02}:59:59".format(pgrecs['enddate'][i], dhour)
|
|
1756
|
+
|
|
1757
|
+
#
|
|
1758
|
+
# check if valid host to process update control
|
|
1759
|
+
#
|
|
1760
|
+
def valid_control_host(cstr, hosts, logact):
|
|
1761
|
+
|
|
1762
|
+
host = PgLOG.get_host(1)
|
|
1763
|
+
if hosts:
|
|
1764
|
+
if re.search(host, hosts, re.I):
|
|
1765
|
+
if hosts[0] == '!':
|
|
1766
|
+
return PgLOG.pglog("{}: CANNOT be processed on {}".format(cstr, hosts[1:]), logact)
|
|
1767
|
+
elif not re.match(r'^!', hosts):
|
|
1768
|
+
return PgLOG.pglog("{}-{}: MUST be processed on {}".format(host, cstr, hosts), logact)
|
|
1769
|
+
|
|
1770
|
+
return PgLOG.SUCCESS
|
|
1771
|
+
|
|
1772
|
+
#
|
|
1773
|
+
# reset updated data time
|
|
1774
|
+
#
|
|
1775
|
+
def reset_data_time(qu, ddate, dhour, lidx):
|
|
1776
|
+
|
|
1777
|
+
pgrec = PgOPT.PGOPT['UCNTL']
|
|
1778
|
+
record = {'chktime' : int(time.time())}
|
|
1779
|
+
if ddate:
|
|
1780
|
+
if dhour is None: dhour = 0 if qu == 'H' else 23
|
|
1781
|
+
dtime = "{} {:02}:59:59".format(ddate, dhour)
|
|
1782
|
+
if lidx not in PgOPT.PGOPT['DTIMES'] or PgUtil.pgcmp(PgOPT.PGOPT['DTIMES'][lidx], dtime) < 0:
|
|
1783
|
+
PgOPT.PGOPT['DTIMES'][lidx] = dtime
|
|
1784
|
+
|
|
1785
|
+
# get earliest data time
|
|
1786
|
+
for ltime in PgOPT.PGOPT['DTIMES'].values():
|
|
1787
|
+
if PgUtil.pgcmp(ltime, dtime) < 0: dtime = ltime
|
|
1788
|
+
|
|
1789
|
+
if not pgrec['datatime'] or PgUtil.pgcmp(pgrec['datatime'], dtime) < 0:
|
|
1790
|
+
PgOPT.PGOPT['UCNTL']['datatime'] = record['datatime'] = dtime
|
|
1791
|
+
|
|
1792
|
+
if PgDBI.pgupdt("dcupdt", record, "cindex = {}".format(pgrec['cindex']), PgOPT.PGOPT['extlog']) and 'datatime' in record:
|
|
1793
|
+
PgLOG.pglog("{}-C{}: Data time updated to {}".format(PgOPT.params['DS'], pgrec['cindex'], dtime), PgOPT.PGOPT['emllog'])
|
|
1794
|
+
|
|
1795
|
+
#
|
|
1796
|
+
# adjust control time according to the control offset
|
|
1797
|
+
#
|
|
1798
|
+
def adjust_control_time(cntltime, freq, unit, offset, curtime):
|
|
1799
|
+
|
|
1800
|
+
if offset:
|
|
1801
|
+
ofreq = get_control_time(offset, "Control Offset")
|
|
1802
|
+
if ofreq: # remove control offset
|
|
1803
|
+
nfreq = ofreq.copy()
|
|
1804
|
+
for i in range(6):
|
|
1805
|
+
if nfreq[i]: nfreq[i] = -nfreq[i]
|
|
1806
|
+
cntltime = PgUtil.adddatetime(cntltime, nfreq[0], nfreq[1], nfreq[2], nfreq[3], nfreq[4], nfreq[5], nfreq[6])
|
|
1807
|
+
else:
|
|
1808
|
+
ofreq = None
|
|
1809
|
+
|
|
1810
|
+
(cdate, ctime) = re.split(' ', cntltime)
|
|
1811
|
+
|
|
1812
|
+
if unit == "H":
|
|
1813
|
+
hr = 0
|
|
1814
|
+
if ctime:
|
|
1815
|
+
ms = re.match(r'^(\d+)', ctime)
|
|
1816
|
+
if ms: hr = int(int(ms.group(1))/freq[3])*freq[3]
|
|
1817
|
+
else:
|
|
1818
|
+
i = 0
|
|
1819
|
+
cntltime = "{} {:02}:00:00".format(cdate, hr)
|
|
1820
|
+
else:
|
|
1821
|
+
cdate = PgUtil.enddate(cdate, (0 if unit == "W" else 1), unit, freq[6])
|
|
1822
|
+
cntltime = "{} 00:00:00".format(cdate)
|
|
1823
|
+
|
|
1824
|
+
if ofreq: cntltime = PgUtil.adddatetime(cntltime, ofreq[0], ofreq[1], ofreq[2], ofreq[3], ofreq[4], ofreq[5], ofreq[6]) # add control offset
|
|
1825
|
+
while PgUtil.pgcmp(cntltime, curtime) <= 0:
|
|
1826
|
+
cntltime = PgUtil.adddatetime(cntltime, freq[0], freq[1], freq[2], freq[3], freq[4], freq[5], freq[6])
|
|
1827
|
+
|
|
1828
|
+
return cntltime
|
|
1829
|
+
|
|
1830
|
+
#
|
|
1831
|
+
# reset control time
|
|
1832
|
+
#
|
|
1833
|
+
def reset_control_time():
|
|
1834
|
+
|
|
1835
|
+
pgrec = PgOPT.PGOPT['UCNTL']
|
|
1836
|
+
cstr = "{}-C{}".format(PgOPT.params['DS'], pgrec['cindex'])
|
|
1837
|
+
|
|
1838
|
+
gmt = PgLOG.PGLOG['GMTZ']
|
|
1839
|
+
PgLOG.PGLOG['GMTZ'] = 0
|
|
1840
|
+
curtime = PgUtil.curtime(1)
|
|
1841
|
+
PgLOG.PGLOG['GMTZ'] = gmt
|
|
1842
|
+
|
|
1843
|
+
(freq, unit) = PgOPT.get_control_frequency(pgrec['frequency'])
|
|
1844
|
+
if not freq: return PgLOG.pglog("{}: {}".format(cstr, unit), PgOPT.PGOPT['emlerr'])
|
|
1845
|
+
cntltime = PgUtil.check_datetime(pgrec['cntltime'], curtime)
|
|
1846
|
+
nexttime = adjust_control_time(cntltime, freq, unit, pgrec['cntloffset'], curtime)
|
|
1847
|
+
if PgLOG.PGLOG['ERRCNT']:
|
|
1848
|
+
cfreq = get_control_time(pgrec['retryint'], "Retry Interval")
|
|
1849
|
+
if cfreq:
|
|
1850
|
+
while PgUtil.pgcmp(cntltime, curtime) <= 0:
|
|
1851
|
+
cntltime = PgUtil.adddatetime(cntltime, cfreq[0], cfreq[1], cfreq[2], cfreq[3], cfreq[4], cfreq[5], cfreq[6])
|
|
1852
|
+
|
|
1853
|
+
if PgUtil.pgcmp(cntltime, nexttime) < 0: nexttime = cntltime
|
|
1854
|
+
|
|
1855
|
+
record = {}
|
|
1856
|
+
cstr += ": Next Control Time "
|
|
1857
|
+
if not pgrec['cntltime'] or PgUtil.pgcmp(nexttime, pgrec['cntltime']) > 0:
|
|
1858
|
+
record['cntltime'] = nexttime
|
|
1859
|
+
cstr += "set to {}".format(nexttime)
|
|
1860
|
+
if PgLOG.PGLOG['ERRCNT']: cstr += " to retry"
|
|
1861
|
+
else:
|
|
1862
|
+
cstr += "already set to {}".format(pgrec['cntltime'])
|
|
1863
|
+
|
|
1864
|
+
cstr += " for Action {}({})".format(PgOPT.PGOPT['CACT'], PgOPT.OPTS[PgOPT.PGOPT['CACT']][1])
|
|
1865
|
+
record['pid'] = 0
|
|
1866
|
+
if PgDBI.pgupdt("dcupdt", record, "cindex = {}".format(pgrec['cindex']), PgOPT.PGOPT['extlog']):
|
|
1867
|
+
PgLOG.pglog(cstr, PgOPT.PGOPT['emllog'])
|
|
1868
|
+
|
|
1869
|
+
#
|
|
1870
|
+
# get array information of individual controlling time
|
|
1871
|
+
#
|
|
1872
|
+
def get_control_time(val, type):
|
|
1873
|
+
|
|
1874
|
+
if not val or val == '0': return 0
|
|
1875
|
+
if re.search(r'/(\d+)$', val):
|
|
1876
|
+
return PgLOG.pglog("{}: '{}' NOT support Fraction".format(val, type), PgOPT.PGOPT['emlerr'])
|
|
1877
|
+
|
|
1878
|
+
ctimes = [0]*7 # initialize control times
|
|
1879
|
+
ms = re.search(r'(\d+)Y', val, re.I)
|
|
1880
|
+
if ms: ctimes[0] = int(ms.group(1))
|
|
1881
|
+
ms = re.search(r'(\d+)M', val, re.I)
|
|
1882
|
+
if ms: ctimes[1] = int(ms.group(1))
|
|
1883
|
+
ms = re.search(r'(\d+)D', val, re.I)
|
|
1884
|
+
if ms: ctimes[2] = int(ms.group(1))
|
|
1885
|
+
ms = re.search(r'(\d+)W', val, re.I)
|
|
1886
|
+
if ms: ctimes[2] += 7*int(ms.group(1))
|
|
1887
|
+
ms = re.search(r'(\d+)H', val, re.I)
|
|
1888
|
+
if ms: ctimes[3] = int(ms.group(1))
|
|
1889
|
+
ms = re.search(r'(\d+)N', val, re.I)
|
|
1890
|
+
if ms: ctimes[4] = int(ms.group(1))
|
|
1891
|
+
ms = re.search(r'(\d+)S', val, re.I)
|
|
1892
|
+
if ms: ctimes[5] = int(ms.group(1))
|
|
1893
|
+
|
|
1894
|
+
for ctime in ctimes:
|
|
1895
|
+
if ctime > 0: return ctimes
|
|
1896
|
+
|
|
1897
|
+
return PgLOG.pglog("{}: invalid '{}', must be (Y,M,W,D,H,N,S)".format(val, type), PgOPT.PGOPT['emlerr'])
|
|
1898
|
+
|
|
1899
|
+
#
|
|
1900
|
+
# get group index from given option string
|
|
1901
|
+
#
|
|
1902
|
+
def get_group_index(option, edate, ehour, freq):
|
|
1903
|
+
|
|
1904
|
+
ms = re.search(r'-GI\s+(\S+)', option, re.I)
|
|
1905
|
+
if ms: return int(replace_pattern(ms.group(1), edate, ehour, freq))
|
|
1906
|
+
|
|
1907
|
+
ms = re.search(r'-GN\s+(.*)$', option, re.I)
|
|
1908
|
+
if ms:
|
|
1909
|
+
grp = ms.group(1)
|
|
1910
|
+
if grp[0] == "'":
|
|
1911
|
+
grp = grp[1:]
|
|
1912
|
+
idx = grp.find("'")
|
|
1913
|
+
grp = grp[:idx]
|
|
1914
|
+
else:
|
|
1915
|
+
ms = re.match(r'^(\S+)', grp)
|
|
1916
|
+
if ms: grp = ms.group(1)
|
|
1917
|
+
|
|
1918
|
+
pgrec = PgDBI.pgget("dsgroup", "gindex", "dsid = '{}' AND grpid = '{}'".format(PgOPT.params['DS'], replace_pattern(grp, edate, ehour, freq)), PgOPT.PGOPT['extlog'])
|
|
1919
|
+
if pgrec: return pgrec['gindex']
|
|
1920
|
+
|
|
1921
|
+
return 0
|